Exemple #1
1
 def selector():
     sel = DefaultSelector()
     sel.register(proc.stdout, EVENT_READ, 0)
     sel.register(proc.stderr, EVENT_READ, 1)
     while True:
         for key, mask in sel.select():
             yield key.data, decode(key.fileobj.readline(linesize))
def redirect(fd, callback):
    save_fd = os.dup(fd)

    s_in, s_out = socket.socketpair()
    os.dup2(s_in.fileno(), fd)
    os.close(s_in.fileno())
    s_out.setblocking(False)

    sel = DefaultSelector()
    sel.register(s_out, EVENT_READ)

    def target():
        while running:
            _, = sel.select()  # There's only one event.
            callback(s_out.recv(4096))

    running = True
    thread = Thread(target=target, daemon=True)
    thread.start()

    try:
        yield
    finally:
        running = False
        os.dup2(save_fd, fd)
        os.close(save_fd)
Exemple #3
0
 def selector():
     sel = DefaultSelector()
     sel.register(proc.stdout, EVENT_READ, 0)
     sel.register(proc.stderr, EVENT_READ, 1)
     while True:
         ready = sel.select(line_timeout)
         if not ready and line_timeout:
             raise ProcessLineTimedOut("popen line timeout expired", getattr(proc, "argv", None), getattr(proc, "machine", None))
         for key, mask in ready:
             yield key.data, decode(key.fileobj.readline(linesize))
class Loop:
    def __init__(self):
        self.ready = deque()
        self.selector = DefaultSelector()
        self.futures = {}
    def create_task(self, task):
        self.ready.append(task)
    def run_forever(self):
        while True:

            while not self.ready:
                completed_futures = [future for future in self.futures if not future.running()]
                for future in completed_futures:
                    self.ready.append(self.futures.pop(future))

                # so select() is blocking. If set a negative time out it won't block.
                events = self.selector.select(-1)
                # add these socket events and unregister them from listened to:
                for key, _ in events:
                    self.ready.append(key.data) # add the task to the ready queue
                    self.selector.unregister(key.fileobj)

            while self.ready:
                self.current_task = self.ready.popleft()
                # try to run current_task...
                try:
                    # run task to next yield point
                    reason, what = self.current_task.send(None)
                    # reason, what = self.current_task.send(None)
                    if reason == 'waiting_to_accept':
                        self.selector.register(what, EVENT_READ, self.current_task)
                    elif reason == 'waiting_to_read':
                        self.selector.register(what, EVENT_READ, self.current_task)
                    elif reason == 'waiting_to_write':
                        self.selector.register(what, EVENT_WRITE, self.current_task)
                    elif reason == 'waiting_for_future':
                        self.futures[what] = self.current_task
                    else:
                        raise RuntimeError(
                            'Something bad happened... er. reason={}'.format(reason))
                except StopIteration:
                    pass

    async def sock_recv(self, sock, maxbytes):
        # wait to read from the socket
        await read_wait(sock)
        return sock.recv(maxbytes)
    async def sock_accept(self, sock):
        # wait to read/hear from the socket
        await accept_wait(sock)
        return sock.accept()
    async def sock_sendall(self, sock, data):
        while data:
            # wait to be able to write to the socket
            await write_wait(sock)
            nsent = sock.send(data)
            data = data[nsent:]

    def run_in_executor(self, executor, func, *args):
        return executor.submit(func, *args)
class Loop:
    def __init__(self):
        self.sel = DefaultSelector()
        self.queue = deque()

    def create_task(self, coro):
        sock = coro.send(None)
        self.sel.register(sock, EVENT_WRITE, coro)
        self.queue.append(coro)

    def run(self):
        while self.queue:
            events = self.sel.select()
            for key, _ in events:
                coro = key.data
                event = key.events
                file_obj = key.fileobj
                self.sel.unregister(file_obj)
                try:
                    if event == EVENT_WRITE:
                        self.sel.register(file_obj, EVENT_READ, coro)
                    elif event == EVENT_READ:
                        self.sel.register(file_obj, EVENT_WRITE, coro)
                    coro.send(None)
                except StopIteration:
                    self.queue.popleft()
                except:
                    pass
        else: return
Exemple #6
0
class IRC(Service):
	DEFAULT_PORT = 6667
	TIMEOUT = 3
	def __init__(self,events):
		super().__init__(events,'irc')
		self.servers = set()
		self.selector = DefaultSelector()
		self.events = events
		self.has_servers = Event()
		self.channels = {}#server:[channels]
	def respond(self,addr,command):
		if command['action']=='connect':
			server = command['host']
			nick = command['nick']
			port = command.get('port',self.DEFAULT_PORT)
			self.connect((server,port),nick)
		if command['action']=='status':
			print(self.servers)
			self.broadcast({
				'kind':'status',
				'status':{
					'servers':{
						s: {
							'channels':v
						} for s,v in self.channels.items()
					}
				}
			})
	def connect(self,addr,nick):
		host = addr[0]
		if host not in self.servers:
			print(nick)
			sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			print(addr)
			sock.connect(addr)
			self.events.listen(host,self.check_channels)
			self.selector.register(sock,EVENT_READ|EVENT_WRITE,ChatServerInterface(addr[0],nick,self.events))
			self.servers.add(host)
			self.channels[host] = list()
			if not self.has_servers.is_set():
				self.has_servers.set()
	def check_channels(self,addr,event):
		if event['kind']=='irc':
			if event['command']=='332':
				print(event['event'],event['arguments'])
				self.channels[event['event']].append(event['arguments'][1].lower())
	def loop(self):
		while True:
			self.has_servers.wait()
			for key,mask in self.selector.select():
				if mask&EVENT_READ==EVENT_READ:
					key.data.read_data(key.fileobj)
				if mask&EVENT_WRITE==EVENT_WRITE:
					key.data.write_data(key.fileobj)
Exemple #7
0
def get(path, jobs):
    jobs += 1
    selector = DefaultSelector()

    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    client.setblocking(False)

    try:
        client.connect(('localhost', 5000))
    except BlockingIOError:
        pass

    f = Future()
    selector.register(client.fileno(), EVENT_READ, f)
 def loop():
     pre_loop_barrier.wait(1)
     selector = DefaultSelector()
     selector.register(sig_r_fd, EVENT_READ)
     with suppress(StopIteration):
         while True:
             events = selector.select(timeout=5)
             if len(events) == 0:
                 raise StopIteration
             for key, mask in events:
                 if key.fd == sig_r_fd:
                     data = os.read(sig_r_fd, 4)
                     self._sigchld_caught = True
                     raise StopIteration
     self._post_loop_barrier.wait(1)
Exemple #9
0
 def __init__(self):
     """
     Конструктор событийного цикла, который хранит очередь
     :return:
     """
     self.ready = deque()
     self.selector = DefaultSelector()
Exemple #10
0
	def __init__(self,events):
		super().__init__(events,'irc')
		self.servers = set()
		self.selector = DefaultSelector()
		self.events = events
		self.has_servers = Event()
		self.channels = {}#server:[channels]
    def __init__(self, port):
        self.port = port
        self.sock = None
        self.selector = DefaultSelector()

        # Advance self.listen() method. A future has been attached to the
        # server socket, that has been registered in selector. And a callback
        # has been attached to this future.
        Task(self.listen())
Exemple #12
0
 def __init__(self, port, callback):
     logging.basicConfig(level=logging.INFO)
     self.clients_kept_for_sending = []
     self.selector = DefaultSelector()
     self.callback = callback
     self.server_socket = socket.socket(family=socket.AF_INET6,
         type=socket.SOCK_STREAM)
     self.server_socket.bind(("::0", port))
     self.server_socket.listen(5)
     self.selector.register(self.server_socket, EVENT_READ, data=None)
     self.server_socket.setblocking(False)
class Loop:
    def __init__(self):
        self.ready = deque()
        self.selectors = DefaultSelector() # a way to watch socket IO
    async def sock_recv(self, sock, maxbytes):
        await read_wait(sock) # wait for something happen
        return sock.recv(maxbytes)
    async def sock_accept(self, sock):
        await read_wait(sock)
        return sock.sock_accept()
    async def sock_sendall(self, sock, data):
        while data :
            try:
                nsent = sock.send(data)
                data = data[nsent:] # send partial data
            except BlockingIOError:
                await write_wait(sock)
    def create_task(self, coro):
        self.ready.append(coro)
    def run_forever(self):
        while True:
            while not self.ready:
                events = self.selectors.select()
                for key, _  in events:
                    self.ready.append(key.data)
                    self.selectors.unregister(key.fileobj)
        while self.ready:
            self.current_task = self.ready.popleft()
            try:
                op, *args = self.current_task.send(None) # run to the yield

                getattr(self,op)(*args) # Sneaky method call

            except StopIteration:
                pass
    def read_wait(self, sock):
        self.selectors.register(sock, EVENT_READ, self.current_task)
    def write_wait(self, sock):
        self.selectors.register
class Loop:
    def __init__(self):
        self.ready = deque()
        self.selector = DefaultSelector()
    def create_task(self, task):
        self.ready.append(task)
    def run_forever(self):
        while True:
            # hmmn, nothing to run -> must be waiting on stuff...
            while not self.ready:
                events = self.selector.select()
                # add these events and unregister them from listened to:
                for key, _ in events:
                    self.ready.append(key.data)
                    self.selector.unregister(key.fileobj)
            while self.ready:
                self.current_task = self.ready.popleft()
                # try to run current_task...
                try:
                    # run task to next yield point
                    reason, sock = next(self.current_task)
                    if reason == 'waiting_to_accept':
                        self.selector.register(sock, EVENT_READ, self.current_task)
                    elif reason == 'waiting_to_read':
                        self.selector.register(sock, EVENT_READ, self.current_task)
                    elif reason == 'waiting_to_write':
                        self.selector.register(sock, EVENT_WRITE, self.current_task)
                    else:
                        raise RuntimeError('Something bad happened... er. reason={}'.format(reason))
                except StopIteration:
                    pass

    def sock_recv(self, sock, maxbytes):
        # wait to read from the socket
        return sock.recv(maxbytes)
    def sock_accept(self, sock):
        # wait to read/hear from the socket
        return sock.accept()
    def sock_sendall(self, sock, data):
        while data:
            # wait to be able to write to the socket
            nsent = sock.send(data)
            data = data[nsent:]
    def __init__(self, fb):
        assert isinstance(fb, Framebuffer)
        self._fb = fb

        # get all input devices
        devices = [InputDevice(fn) for fn in evdev.list_devices()]
        # filter out non key devices
        for device in devices.copy():
            cap = device.capabilities()
            if ecodes.EV_KEY not in cap:
                devices.remove(device)
                continue

        self._selector = DefaultSelector()
        # This works because InputDevice has a `fileno()` method.
        for device in devices:
            self._selector.register(device, EVENT_READ)
Exemple #16
0
 def __init__(self):
     self.selector = DefaultSelector()
     self.init_hardware()
     self.mouseevents = []
     self.prev_value = 1
     self.keynames = {
         127: "menu",
         28: "ok",
         103: "up",
         108: "down",
         105: "left",
         106: "right",
         1: "back",
         172: "home",
         114: "volumedown",
         115: "volumeup",
         116: "power",
         272: "btn_left"}
Exemple #17
0
class Loop:
    def __init__(self):
        self.ready = deque()
        self.selector = DefaultSelector()

    async def sock_recv(self, sock, maxbytes):
        await read_wait(sock)
        return sock.recv(maxbytes)

    async def sock_accept(self, sock):
        await read_wait(sock)
        return sock.accept()

    async def sock_sendall(self, sock, data):
        while data:
            try:
                nsent = sock.send(data)
                data = data[nsent:]
            except BlockingIOError:
                await write_wait(sock)

    def create_task(self, coro):
        self.ready.append(coro)

    def run_forever(self):
        while True:
            while not self.ready:
                events = self.selector.select()
                for key, _ in events:
                    self.ready.append(key.data)
                    self.selector.unregister(key.fileobj)

            while self.ready:
                self.current_task = self.ready.popleft()
                try:
                    op, *args = self.current_task.send(None)
                    getattr(self, op)(*args)
                except StopIteration:
                    pass
                
    def read_wait(self, sock):
        self.selector.register(sock, EVENT_READ, self.current_task)

    def write_wait(self, sock):
        self.selector.register(sock, EVENT_WRITE, self.current_task)
Exemple #18
0
class Connection(LibConnection):

    # Reference to the handling app on the side of the server
    TYPE_HANDLER = 'client-handler'

    LL_EVENT_ACCEPT = 'accept'
    LL_EVENT_READ = 'read'
    LL_EVENT_WRITE = 'write'
    LL_EVENT_DISCONNECTED = 'disconnected'

    _sel: DefaultSelector = None

    _perform_select: bool = True
    _subscribed_events: dict = None

    _t_selecting_events: Thread = None
    _t_read: Thread = None
    _t_write: Thread = None

    _read_lock: Lock = None
    _write_lock: Lock = None

    def __init__(self, *args, **kwargs):
        super(Connection, self).__init__(*args, **kwargs)
        self._subscribed_events = {
            self.LL_EVENT_ACCEPT: list(),
            self.LL_EVENT_READ: list(),
            self.LL_EVENT_WRITE: list(),
            self.LL_EVENT_DISCONNECTED: list(),
        }

    def unsubscribe_from_event(self, event_name: str, cb: callable) -> bool:
        if event_name not in self._subscribed_events:
            return False
        if cb in self._subscribed_events[event_name]:
            self._subscribed_events[event_name].remove(cb)
        return True

    def subscribe_to_event(self, event_name: str, cb: callable) -> bool:
        """
		Subscribing to low-level communication events (accept, read, write).
		:param event_name: One of: 'accept', 'read' or 'write'
		:param cb: Callback that should be used for the subscription
		:return: Returns True in case of successful subscription or False otherwise (Basically indication if subscribed)
		"""
        if event_name not in self._subscribed_events:
            return False

        if cb not in self._subscribed_events[event_name]:
            self._subscribed_events[event_name].append(cb)

        # Already subscribed
        return True

    def _after_open_connection(self):
        super(Connection, self)._after_open_connection()
        self._register_events()
        self._t_selecting_events = Thread(target=self._selecting_events,
                                          daemon=True)

    def ready_to_operate(self):
        """
		This method must be explicitly invoked after reconnection.
		Otherwise the reading/writing to socket will work incorrectly
		:return:
		"""
        self._t_selecting_events.start()

    def _reading(self):
        self._read_lock.acquire()
        while self._is_connected:
            # print('R')
            self._read_lock.acquire()
            self._invoke_events_callbacks(self.LL_EVENT_READ)
        if self._read_lock.locked():
            self._read_lock.release()

    def _writing(self):
        self._write_lock.acquire()
        # print('[W2]', self._is_connected)

        while self._is_connected:
            # print('[W]')
            self._write_lock.acquire()
            self._invoke_events_callbacks(self.LL_EVENT_WRITE)
        if self._write_lock.locked():
            self._write_lock.release()

    def _register_events(self):
        self._sel = DefaultSelector()
        if self._type == self.TYPE_SERVER:
            self._sel.register(self._socket, EVENT_READ, self._event_accept)
        if self._type in (self.TYPE_HANDLER, self.TYPE_CLIENT):
            self._sel.register(self._socket, EVENT_READ | EVENT_WRITE,
                               self._event_communicate)

            self._read_lock = Lock(
            ) if self._read_lock is None else self._read_lock
            self._write_lock = Lock(
            ) if self._write_lock is None else self._write_lock

            self._t_read = Thread(target=self._reading, daemon=True)
            self._t_read.start()

            self._t_write = Thread(target=self._writing, daemon=True)
            self._t_write.start()

    def _disconnected(self):
        print('Disconnected happened')
        self._unregister_events()

    def _unregister_events(self):
        # self._sel = DefaultSelector()
        if self._perform_select:
            # print('Unregistering stuff')
            self._perform_select = False

            self._sel.unregister(self._socket)
            if self._read_lock and self._read_lock.locked():
                self._read_lock.release()
            if self._write_lock and self._write_lock.locked():
                self._write_lock.release()

            if self._t_read and self._t_read.is_alive():
                del self._t_read
            if self._t_write and self._t_write.is_alive():
                del self._t_write

    def _selecting_events(self):
        while self._perform_select:
            for key, mask in self._sel.select():
                key.data(mask)

    def _invoke_events_callbacks(self, event_name, *args, **kwargs):
        # print(f'LL EVENT Invoked: {event_name}')
        if self._subscribed_events[event_name]:
            # print('cb: ', self._subscribed_events, event_name)
            for cb in self._subscribed_events[event_name]:
                cb(*args, **kwargs)
        else:
            if event_name == self.LL_EVENT_WRITE:
                # If there is no write subscriptions, this code
                # would decrease unnecessary CPU usage by just a simple timeout.
                # Should not be considered as a hack, though if you have
                # a better solution - please propose.
                # print('No write subscription')
                sleep(10)

    def _event_accept(self, mask):
        if mask & EVENT_READ:
            new_connection = self.accept(self.TYPE_HANDLER)
            self._invoke_events_callbacks(self.LL_EVENT_ACCEPT, new_connection)

    def _event_communicate(self, mask):
        if mask & EVENT_READ:
            if self._read_lock is not None and self._read_lock.locked():
                # print('MREAD')
                self._read_lock.release()
            # Thread(target=self._invoke_events_callbacks, args=(self.LL_EVENT_READ, ), daemon=True).start()
        if mask & EVENT_WRITE:
            if self._write_lock is not None and self._write_lock.locked():
                # print('MWRITE')
                self._write_lock.release()
            # self._invoke_events_callbacks(self.LL_EVENT_WRITE)

    def collect_frame(self):
        frame = None
        try:
            frame = self._gather_frame_from_socket(SimpleFrame)
            if frame is None:
                self._invoke_events_callbacks(self.LL_EVENT_DISCONNECTED)
        except SSLWantReadError as e:
            # todo move to ll
            pass

        if isinstance(frame, SimpleFrame):
            return frame

        return False
Exemple #19
0
import socket
from selectors import DefaultSelector, EVENT_WRITE

selector = DefaultSelector()

sock = socket.socket()
sock.setblocking(False)
sock2 = socket.socket()
sock2.setblocking(False)

try:
    sock.connect(('xkcd.com', 80))
except BlockingIOError:
    pass

try:
    sock2.connect(('xkcd.com', 80))
except BlockingIOError:
    pass


def connected(key, mask):
    selector.unregister(key.fd)
    print('connected!')


selector.register(sock.fileno(), EVENT_WRITE, connected)
selector.register(sock2.fileno(), EVENT_WRITE, connected)


def loop():
Exemple #20
0
class TorSocketLoop(threading.Thread):
    def __init__(self, our_sock, send_func):
        super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno()))
        self._our_sock = our_sock
        self._send_func = send_func
        self._do_loop = True

        self._cntrl_l = threading.Lock()
        self._cntrl_r, self._cntrl_w = socket.socketpair()

        self._selector = DefaultSelector()
        self._selector.register(self._our_sock, EVENT_READ, self._do_recv)
        self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)

    def _do_recv(self, sock):
        try:
            data = sock.recv(1024)
            self._send_func(data)
        except ConnectionResetError:
            logger.debug('Client was badly disconnected...')

    def _do_stop(self, sock):
        self._do_loop = False

    def _cleanup(self):
        with self._cntrl_l:
            logger.debug('Cleanup')
            self._selector.unregister(self._cntrl_r)
            self._cntrl_w.close()
            self._cntrl_r.close()
            self.close_sock()
            self._selector.close()

    @property
    def fileno(self):
        if not self._our_sock:
            return None
        return self._our_sock.fileno()

    def close_sock(self):
        if not self._our_sock:
            return
        self._selector.unregister(self._our_sock)
        self._our_sock.close()
        self._our_sock = None

    def stop(self):
        #  Because stop could be called twice
        with self._cntrl_l:
            logger.debug('Stopping...')
            self._cntrl_w.send(b'\1')

    def run(self):
        logger.debug('Starting...')
        while self._do_loop:
            events = self._selector.select()
            for key, _ in events:
                callback = key.data
                callback(key.fileobj)

        self._cleanup()
        logger.debug('Stopped!')

    def append(self, data):
        self._our_sock.send(data)
    async def run(self):
        l = [False]*4
        
        selector = DefaultSelector()
        selector.register(self.mouse, EVENT_READ)
        selector.register(self.keyboard, EVENT_READ)

        while True:
            for key, mask in selector.select(0):
                device = key.fileobj
                for event in device.read():
                    if event is not None:
                        # mouse movement
                        if event.type == ecodes.EV_REL:
                            if event.code == ecodes.REL_X:
                                dx = event.value
                                self.controller_state.axis_state.dx += dx
                            elif event.code == ecodes.REL_Y:
                                dy = event.value
                                self.controller_state.axis_state.dy += dy
                                self.controller_state.axis_state.sum_y += dy
                            
                        # mouse click
                        elif event.type == ecodes.EV_KEY:
                            pushed = True
                            if event.value == 0x01:#key down
                                pushed = True
                            elif event.value == 0x00:
                                pushed = False
                            
                            #left click -> ZR
                            if event.code == ecodes.BTN_LEFT:
                                self.button_state.zr(pushed)
                                #right click -> ZL
                            elif event.code == ecodes.BTN_RIGHT:
                                self.button_state.zl(pushed)
                                #middle click -> Y
                            if event.code == ecodes.BTN_MIDDLE:
                                self.button_state.y(pushed)

                            #WASD -> up, left, right, down
                            if event.code == ecodes.KEY_W:
                                if pushed:
                                    self.lstick_state.set_up()
                                    l[0] = True
                                else:
                                    if l[2]:
                                        self.lstick_state.set_down()
                                    else:
                                        self.lstick_state.set_v(self.lstick_state._calibration.v_center)
                                    l[0] = False
                            elif event.code == ecodes.KEY_A:
                                if pushed:
                                    self.lstick_state.set_left()
                                    l[1] = True
                                else:
                                    if l[3]:
                                        self.lstick_state.set_right()
                                    else:
                                        self.lstick_state.set_h(self.lstick_state._calibration.h_center)
                                    l[1] = False
                            elif event.code == ecodes.KEY_S:
                                if pushed:
                                    self.lstick_state.set_down()
                                    l[2] = True
                                else:
                                    if l[0]:
                                        self.lstick_state.set_up()
                                    else:
                                        self.lstick_state.set_v(self.lstick_state._calibration.v_center)
                                    l[2] = False
                            elif event.code == ecodes.KEY_D:
                                if pushed:
                                    self.lstick_state.set_right()
                                    l[3] = True
                                else:
                                    if l[1]:
                                        self.lstick_state.set_up()
                                    else:
                                        self.lstick_state.set_h(self.lstick_state._calibration.h_center)
                                    l[3] = False

                                #FVE -> a,b,x
                            elif event.code == ecodes.KEY_F:
                                self.button_state.a(pushed)
                            elif event.code == ecodes.KEY_V:
                                self.button_state.b(pushed)
                            elif event.code == ecodes.KEY_E:
                                self.button_state.x(pushed)

                                #TG -> rstick, lstick
                            elif event.code == ecodes.KEY_T:
                                self.button_state.l_stick(pushed)
                            elif event.code == ecodes.KEY_G:
                                self.button_state.r_stick(pushed)
                                #QR -> l,r
                            elif event.code == ecodes.KEY_Q:
                                self.button_state.l(pushed)
                            elif event.code == ecodes.KEY_R:
                                self.button_state.r(pushed)
                            #ZX -> plus, minus
                            elif event.code == ecodes.KEY_Z:
                                self.button_state.minus(pushed)
                            elif event.code == ecodes.KEY_X:
                                self.button_state.plus(pushed)
                            #TAB -> zl
                            elif event.code == ecodes.KEY_TAB:
                                self.button_state.zl(pushed)
                            #SPACE -> b
                            elif event.code == ecodes.KEY_SPACE:
                                self.button_state.b(pushed)
                                #esc -> home
                            elif event.code == ecodes.KEY_ESC:
                                self.button_state.home(pushed)
                                
                            #del -> break
                            elif event.code == ecodes.KEY_DELETE:
                                self.mouse.ungrab()
                                return
                        
                try:
                    await self.controller_state.send()
                except NotConnectedError:
                    logger.info('Connection was lost.')
                    self.mouse.ungrab()
                    return
Exemple #22
0
class Loop:
    """
    Событийный цикл, который использует очередь сдвухсторонним доступом
    Цикл опрашивает селектор на предмет готовности событий для чтения и записи
    в сокет
    """
    def __init__(self):
        """
        Конструктор событийного цикла, который хранит очередь
        :return:
        """
        self.ready = deque()
        self.selector = DefaultSelector()

    async def sock_recv(self, sock, max_bytes):
        """
        курутина для чтения данных из сокета в асинхронный способ
        :param sock: дескриптор сокета
        :param max_bytes: максимальное количество байт, который могут быть
        прочитаны за один раз без блокировки
        :return: принятые из сокета данные в банарном виде
        """
        await read_wait(sock)
        return sock.recv(max_bytes)

    async def sock_accept(self, sock):
        await read_wait(sock)
        return sock.accept()

    async def sock_sendall(self, sock, data):
        while data:
            try:
                n_sent = sock.send(data)
                data = data[n_sent:]
            except BlockingIOError:
                await write_wait(sock)

    def create_task(self, coro):
        self.ready.append(coro)

    def run_forever(self):
        while True:
            while not self.ready:
                events = self.selector.select()
                for key, _ in events:
                    self.ready.append(key.data)
                    self.selector.unregister(key.fileobj)

            while self.ready:
                self.current_task = self.ready.popleft()
                try:
                    # запускаем генератор до появления yield
                    op, *args = self.current_task.send(None)
                    getattr(self, op)(*args)
                except StopIteration:
                    pass

    def read_wait(self, sock):
        self.selector.register(sock, EVENT_READ, self.current_task)

    def write_wait(self, sock):
        self.selector.register(sock, EVENT_WRITE, self.current_task)
Exemple #23
0
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE

selector = DefaultSelector()
stopped = False
urls_todo = {"/", "/1", "/2", "/3"}


class Future:
    def __init__(self):
        self.result = None
        self._callbacks = []

    def add_done_callback(self, fn):
        self._callbacks.append(fn)

    def set_result(self, result):
        self.result = result
        for fn in self._callbacks:
            fn(self)

    def __iter__(self):
        """
        yield的出现使得__iter__函数变成一个生成器,生成器本身就有next方法,所以不需要额外实现。
        yield from x语句首先调用iter(x)获取一个迭代器(生成器也是迭代器)
        """
        yield self  # 外面使用yield from把f实例本身返回
        return self.result, 'x'  # 在Task.step中send(result)的时候再次调用这个生成器,但是此时会抛出stopInteration异常,并且把self.result返回


def read(sock):
Exemple #24
0
class TorReceiver(threading.Thread):
    def __init__(self, tor_socket, handler_mgr):
        super().__init__(name='RecvLoop_{}'.format(tor_socket.ip_address[0:7]))

        self._tor_socket = tor_socket

        self._handler_mgr = handler_mgr
        self._do_loop = False

        # fmt: off
        self._regs_funcs_map = {
            'reg': {
                socket.socket: self.register_socket,
                TorStream: self.register_stream
            },
            'unreg': {
                socket.socket: self.unregister_socket,
                TorStream: self.unregister_stream
            }
        }
        # fmt: on
        self._stream_to_callback = {}
        self._selector = DefaultSelector()

        self._cntrl_r, self._cntrl_w = socket.socketpair()
        self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
        self._selector.register(self._tor_socket.ssl_socket, EVENT_READ,
                                self._do_recv)

    def _cleanup(self):
        self._selector.unregister(self._cntrl_r)
        self._cntrl_w.close()
        self._cntrl_r.close()
        self._selector.unregister(self._tor_socket.ssl_socket)
        self._selector.close()

    def start(self):
        self._do_loop = True
        super().start()

    def stop(self):
        logger.debug('Stopping receiver thread...')
        self._cntrl_w.send(b'\1')
        self.join()

    def register(self, sock_or_stream, events, callback):
        func = self._regs_funcs_map['reg'].get(type(sock_or_stream))
        if not func:
            raise Exception('Unknown object for register')
        return func(sock_or_stream, events, callback)

    def register_socket(self, sock, events, callback):
        return self._selector.register(sock, events, callback)

    def register_stream(self, stream: TorStream, events, callback):
        if events & EVENT_WRITE:
            raise Exception('Write event not supported yet')
        stream.register(callback)
        if stream not in self._stream_to_callback:
            self._stream_to_callback[stream] = []
        self._stream_to_callback[stream].append(callback)

    def unregister(self, sock_or_stream):
        func = self._regs_funcs_map['unreg'].get(type(sock_or_stream))
        if not func:
            raise Exception('Unknown object for unregister')
        return func(sock_or_stream)

    def unregister_socket(self, sock):
        return self._selector.unregister(sock)

    def unregister_stream(self, stream):
        callbacks = self._stream_to_callback.pop(stream, None)
        if not callbacks:
            raise Exception('There is no such stream registered')
        for callback in callbacks:
            stream.unregister(callback)

    def _do_stop(self, raw_socket, mask):
        self._do_loop = False

    def _do_recv(self, raw_socket, mask):
        for cell in self._tor_socket.recv_cell_async():
            logger.debug('Cell received: %r', cell)
            try:
                self._handler_mgr.handle(cell)
            except BaseException:
                logger.exception('Some handle errors')

    def run(self):
        logger.debug('Starting...')
        while self._do_loop:
            events = self._selector.select()
            for key, mask in events:
                callback = key.data
                callback(key.fileobj, mask)

        self._cleanup()
        logger.debug('Stopped...')
Exemple #25
0
 def __init__(self):
     self.address = ('127.0.0.1', 8080)
     self.selector = DefaultSelector()
     self.decoder = DecoderHandler()
     self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
class Crawler:
    select = DefaultSelector()
    finished = False

    def __init__(self, url):
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client.setblocking(False)
        self.url = url
        self.response = b''
        global urls

    # getUrl 这个委派生成器中创建的了3个yield from 通道用于调用方和3个子生成器的通信。
    # 但是这3个通道不是同时建立的,而是先建立一个完了再建立下一个,同一时刻只有一个通道存在。也就是说同一时刻调用方只和1个子生成器通信(同一时刻调用方只运行1个协程)
    def getUrl(self):
        self.__parseUrl()

        # 先暂停执行连接
        yield from self.__connect()

        # 连接完成后暂停执行发请求
        yield from self.__sendReq()

        # 发送请求后执行接收响应
        yield from self.__recvResponse()

    def __connect(self):
        try:
            self.client.connect((self.host, 80))
        except:
            pass

        f = Future()

        self.select.register(self.client, EVENT_WRITE, data=f)

        yield f

    def __sendReq(self):
        self.select.unregister(self.client)

        self.client.send(self.send_msg.encode('utf-8'))

        f = Future()

        self.select.register(self.client, EVENT_READ, data=f)

        yield f

    def __recvResponse(self):
        while True:
            self.select.unregister(self.client)
            data = self.client.recv(1024)

            if data:
                self.response += data
                f = Future()
                self.select.register(self.client, EVENT_READ, data=f)
                yield f
            else:
                self.client.close()

                urls.remove(self.url)

                self.__class__.finished = False if len(urls) else True

                self.__saveHtml()

                break

    def __parseUrl(self):
        url_component = urlparse(self.url)
        self.host = url_component.netloc
        self.path = '/' if url_component.path == '' else url_component.path
        self.send_msg = "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n" % (
            self.path, self.host)

    def __saveHtml(self):
        try:
            dir_path = './crawled_page2/'
            fname = 'index.html' if self.path == '/' else self.path.strip(
                '/').strip('.html') + '.html'
            content_arr = self.response.decode('utf-8').split("\r\n\r\n")
            content_arr[0] = ''
            content = ''.join(content_arr)

            if not os.path.isdir(dir_path):
                os.mkdir(dir_path)

            with open(dir_path + fname, 'w', encoding='utf-8') as f:
                f.write(content)
            print("%s 爬取成功" % str(fname))
        except BaseException as e:
            print(e)

        finally:
            del self

    @classmethod
    def loopEvents(cls):
        while not cls.finished:
            events = cls.select.select()
            for key, mask in events:
                f = key.data
                f.runCallback()
class Main(object):
    def __init__(self, fb):
        assert isinstance(fb, Framebuffer)
        self._fb = fb

        # get all input devices
        devices = [InputDevice(fn) for fn in evdev.list_devices()]
        # filter out non key devices
        for device in devices.copy():
            cap = device.capabilities()
            if ecodes.EV_KEY not in cap:
                devices.remove(device)
                continue

        self._selector = DefaultSelector()
        # This works because InputDevice has a `fileno()` method.
        for device in devices:
            self._selector.register(device, EVENT_READ)

    def _color565(self, r, g, b):
        """Convert red, green, blue components to a 16-bit 565 RGB value. Components
        should be values 0 to 255.
        """
        return (((r & 0xF8) << 8) | ((g & 0xFC) << 3) | (b >> 3))

    def _img_to_rgb565_bytes(self):
        pixels = [self._color565(r, g, b) for (r, g, b) in self._img.getdata()]
        return pack('H' * len(pixels), *pixels)

    def _write_image(self, img):
        if (1 == self._fb.bits_per_pixel):
            image_bytes = img.tobytes("raw", "1;IR", self._fb.line_length)
        else:
            self._img = img
            image_bytes = self._img_to_rgb565_bytes()
        self._fb.write_raw(image_bytes)

    def _do_countdown(self):
        if (1 == self._fb.bits_per_pixel):
            img = Image.new("1", self._fb.resolution, 1)
        else:
            img = Image.new("RGB", self._fb.resolution, 'black')

        fnt = ImageFont.truetype(font="/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", size=96)
        d = ImageDraw.Draw(img)
        for i in range(3, 0, -1):
            # clear the image
            d.rectangle(((0, 0), self._fb.resolution), fill=1)
            # draw the text
            text_width, text_height = d.textsize(str(i), font=fnt)
            x = (self._fb.resolution.x - text_width) / 2
            y = (self._fb.resolution.y - text_height) / 2
            d.text((x, y), str(i), font=fnt)
            self._write_image(img)
            # give some time to read
            time.sleep(1.25)

    def _draw_text(self, text, size):
        if (1 == self._fb.bits_per_pixel):
            img = Image.new("1", self._fb.resolution, 1)
        else:
            img = Image.new("RGB", self._fb.resolution, 'black')
        fnt = ImageFont.truetype(font="/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", size=size)
        d = ImageDraw.Draw(img)
        text_width, text_height = d.textsize(text, font=fnt)
        x = (self._fb.resolution.x - text_width) / 2
        y = (self._fb.resolution.y - text_height) / 2
        d.text((x, y), text, font=fnt)
        self._write_image(img)

    def _take_picture(self):
        args = 'fswebcam --quiet --no-banner'
        args += ' --scale {0}x{1}'.format(self._fb.resolution.x, self._fb.resolution.y)
        if self._fb.grayscale or self._fb.bits_per_pixel == 1:
            args += ' --greyscale'
        args += ' --png --save {0}'.format(self._filename)
        fswebcam = Popen(args, shell=True)
        fswebcam.wait()

    def _show_picture(self):
        if (1 == self._fb.bits_per_pixel):
            self._write_image(ImageMath.eval('convert(img, "1")', img=Image.open(self._filename)))
        else:
            self._write_image(img=Image.open(self._filename))

    def run(self):
        self._draw_text('Ready!', 36)
        # TODO: listen for keypresses on a background thread, otherwise they pile up while taking a picture.
        while True:
            for key, mask in self._selector.select():
                device = key.fileobj
                for event in device.read():
                    if event.type != ecodes.EV_KEY:
                        # ignore non-key events
                        continue
                    if not event.value:
                        # ignore key up events
                        continue
                    if event.code in [ecodes.KEY_ENTER, ecodes.KEY_CAMERA]:
                        # enter button or camera button takes a picture
                        self._do_countdown()
                        self._draw_text('Cheese!', 36)
                        self._filename = 'raw-{0}.png'.format(time.strftime("%Y%m%d-%H%M%S"))
                        self._take_picture()
                        self._draw_text('Please wait...', 24)
                        self._show_picture()
                    elif event.code == ecodes.KEY_BACKSPACE:
                        # back button to exit as usual.
                        exit(0)
Exemple #28
0
class MySocket:
    def get_conn(self):
        self.sel = DefaultSelector()
        self.host_port = ('0.0.0.0', 45633)
        self.ss = socket.socket()
        self.ss.setblocking(False)
        self.ss.bind(self.host_port)
        self.ss.listen(10)
        self.sel.register(self.ss.fileno(), EVENT_READ, self.get_accept)
        while 1:
            ready = self.sel.select()
            for key, mask in ready:
                print(key.data)
                call_back = key.data
                call_back(key)

    def get_accept(self, mask):
        self.sel.unregister(mask.fd)
        self.conn, self.addr = self.ss.accept()
        self.conn.setblocking(False)
        self.sel.register(self.conn.fileno(), EVENT_READ, self.get_recv)

    def get_recv(self, mask):

        # self.sel.unregister(mask.fd)
        self.data = self.conn.recv(1024)
        if self.data:
            # self.conn.send(self.data)
            self.sel.unregister(mask.fd)
            self.sel.register(self.conn.fileno(), EVENT_WRITE, self.get_send)
        else:
            self.conn.close()
            self.sel.unregister(mask.fd)

    def get_send(self, mask):
        self.sel.unregister(mask.fd)
        self.conn.send(self.data)
        self.sel.register(self.conn.fileno(), EVENT_READ, self.get_recv)
Exemple #29
0
def audio(dev):
    for event in dev.read_loop():
        if event.value == 1 and event.type == 1:
            print('按下键', evdev.ecodes.KEY[event.code], 'device:', dev.fn)


def detectInputKey(dev_keyboards):
    while True:
        select([dev], [], [])
        for event in dev.read():
            print("code:{} value:{}".format(event.code, event.value))


if __name__ == '__main__':
    keyboards = check_keyboard()
    selector = DefaultSelector()
    #selector.register(keyboards[0],EVENT_READ)
    #selector.register(keyboards[1],EVENT_READ)
    for kb in keyboards:
        selector.register(kb, EVENT_READ)

    try:
        while True:
            for key, mask in selector.select():
                dev = key.fileobj
                print('有事件发生')
                for event in dev.read():
                    print(event)
                    if event.value == 1 and event.type == 1:
                        pass
                        #print('按下键',evdev.ecodes.KEY[event.code],'device:',dev.fn)
# * non blocking sockets 2°
# * callbacks, allowing multiples operations to be waiting concurrently for i/o operations 3°
# * event loop 3°
# * coroutines 4°
# --> Future
# --> generators
# --> Task, responsible for calling next() on the generators


import socket
import time

# 2° to wait for some event on a non blocking socket
# jusk ask for the default selector for your system
from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ
selector = DefaultSelector()
n_tasks = 0


# a Future represent some pending event we're waiting for
class Future:
    def __init__(self):
        self.callbacks = [] # what to do when event occurs

    def resolve(self):
        for c in self.callbacks:
            c()

class Task:
    def __init__(self, gen):
        self.gen = gen
Exemple #31
0
本代码使用 I/O多路复用 中的select完成http请求 单线程
*事件循环使用场景
    tornado
    twisted  (scrapy、django channels)
    gevent
    asyncio
"""
import socket
from urllib.parse import urlparse
# selectors是对select库的进一步封装
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE

# import select
# select.select()

selector = DefaultSelector()  # win用poll ,linux用epoll
urls = []
stop = False


class Fetcher:
    def connected(self, key):
        selector.unregister(key.fd)  # fd == self.client.fileno()
        self.client.send(
            "GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(
                self.path, self.host).encode("utf8"))
        # 注册一个事件: 先监听等待,当当前socket变为可写EVENT_READ时候,调用self.readable方法
        # (里边有recv data方法)里边的逻辑
        selector.register(self.client.fileno(), EVENT_READ, self.readable)

    def readable(self, key):
Exemple #32
0
 def selector():
     sel = DefaultSelector()
     sel.register(proc.stdout.channel, EVENT_READ)
     while True:
         for key, mask in sel.select():
             yield
Exemple #33
0
 def __init__(self):
     self.tasks = deque()
     self.selector = DefaultSelector()
Exemple #34
0
User-Agent: PPTViPhone/7.8.0.1911 CFNetwork/811.5.4 Darwin/16.7.0
Accept-Encoding: gzip, deflate
Content-Length: 2

ab

'''
host='ulogs.umeng.com'
clientcert=path.join(path.abspath(path.dirname(path.dirname(__file__))),'certs','soul_client.pem')
servercert=path.join(path.abspath(path.dirname(path.dirname(__file__))),'certs','sn_new_server.crt')
serverkey=path.join(path.abspath(path.dirname(path.dirname(__file__))),'certs','sn_new_server_key.pem')




selector=DefaultSelector()
ctx=SSLContext(PROTOCOL_SSLv23)
ctx.load_cert_chain(certfile=clientcert)
sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setblocking(False)
selector.register(sock,EVENT_WRITE)
try:
    sock.connect((host,443))
except BlockingIOError:
    pass

while True:
    events=selector.select(2)
    if events:
        break
selector.unregister(sock)
class TCPServer:

    def __init__(self, port):
        self.port = port
        self.sock = None
        self.selector = DefaultSelector()

        # Advance self.listen() method. A future has been attached to the
        # server socket, that has been registered in selector. And a callback
        # has been attached to this future.
        Task(self.listen())

    def run(self):
        while True:
            ready = self.selector.select()
            for key, events in ready:
                fut = key.data
                fut.resolve()

    def listen(self):
        server_sock = self._create_socket()
        with server_sock:
            try:
                client_sock, remote_addr = self.sock.accept()
            except BlockingIOError:
                pass

            fut = Future()
            self.selector.register(self.sock, EVENT_READ, fut)
            while True:
                yield fut
                Task(self.handle_client())

    def handle_client(self):
        client_sock, remote_addr = self.sock.accept()
        client_sock.setblocking(False)
        client_name = '{} {}'.format(*client_sock.getpeername())
        print('Connected', client_name)

        # Register client socket for reading and wait for this event
        fut = Future()
        self.selector.register(client_sock, EVENT_READ, fut)
        yield fut

        buffer = []
        while True:
            try:
                chunk = client_sock.recv(1024)
                buffer.append(chunk)
            except BlockingIOError:
                # Yield the same future, so we don't need to register
                # it again. By the way, callback is the same for all
                # futures, so it doesn't matter in my case.
                yield fut
                continue

            print('Received chunk from client', client_name)
            if chunk.endswith(b'end'):
                print('Got all data from client', client_name)
                request = b''.join(buffer).decode('utf8')
                response = (request.upper() * 100).encode('utf8')
                self.selector.unregister(client_sock)
                fut = Future()
                self.selector.register(client_sock, EVENT_WRITE, fut)
                break

        # This future was attached to the client_sock
        yield fut
        while True:
            try:
                bytes_sent = client_sock.send(response)
                print(bytes_sent, 'bytes sent to client', client_name)
                response = response[bytes_sent:]
            except BlockingIOError:
                print('Socket blockes for', client_name)
                yield fut
                continue
            else:
                if not response:
                    # All response sent
                    print('Response sent to client', client_name)
                    self.selector.unregister(client_sock)
                    client_sock.close()
                    break

    def _create_socket(self):
        self.sock = socket.socket()
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 10)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind(('', self.port))
        self.sock.setblocking(False)
        self.sock.listen(10)
        self.responses = {}
        return self.sock
Exemple #36
0
 def start(self):
     _sel = Selector()
     for n in self.nodes.values():
         _sel.register(n['sock'], EVENT_READ, {})
     Thread(target=self._loop, args=(_sel, ), daemon=True).start()
Exemple #37
0
# 使用socket获取http报文
# 把setblocking设为false 让学员自己解决问题
import socket
from urllib.parse import urlparse
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE

# 申明select
sel = DefaultSelector()


class Downloader:

    def __init__(self, sel):
        self.domain = ""
        self.url = ""
        self.path = ""
        self.host = ""
        self.selector = sel
        self.ss = ""
        self.data = b""

    def get_client(self):
        # 初始化socket

        self.ss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # 使用非阻塞io,socket不等待连接建立好 直接向下执行
        self.ss.setblocking(False)
        # 虽然申明了一个非阻塞操作,但这个还是阻塞操作,不能交给select
        try:
            self.ss.connect((self.host, 18001))
        except Exception:
Exemple #38
0
#1. epoll并不代表一定比select好
# 在并发高的情况下,连接活跃度不是很高, epoll比select
# 并发性不高,同时连接很活跃, select比epoll好

#通过非阻塞io实现http请求
# select + 回调 + 事件循环
#  并发性高
# 虽然使用单线程,但是省去了I/O等待时线程切换的开销,因为几十个线程切换的开销是很大的,但是成千上万个回调却是很快。这这种方式是高并发编程的核心。

import socket
from urllib.parse import urlparse
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE   # DefaultSelector是选择select、poll还是epoll根据平台自己选择。


selector = DefaultSelector() # 全局变量
#使用select完成http请求
urls = []
stop = False


class Fetcher:
    def connected(self, key):
        selector.unregister(key.fd)  # 注销掉监控的描述符

        print("=========key.fd:", key.fd)
        print("=========self.client.fileno():", self.client.fileno())

        self.client.send("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(self.path, self.host).encode("utf8"))
        selector.register(self.client.fileno(), EVENT_READ, self.readable)  # 注册读取监控

    # 全都是cpu操作,速度远远快于I/O操作,尤其是网络I/O
Exemple #39
0
class Loop:

    def __init__(self):
        self.ready = collections.deque()
        self.selector = DefaultSelector()

    async def sock_accept(self, sock):
        '''
        Wait for the server socket to become readable, accept a new
        client and return her socket, addr pair.
        '''
        await read_wait(sock)
        return sock.accept()

    async def sock_recv(self, sock, maxbytes):
        '''
        Wait for the socket to become readable, read data from the
        socket and return data.
        '''
        await read_wait(sock)
        return sock.recv(maxbytes)

    async def sock_sendall(self, sock, data):
        '''
        While we have some data to send, wait for the socket to become
        writeable and send some data.
        '''
        while data:
            await write_wait(sock)
            nbytes = sock.send(data)
            # Modify data according to the number of bytes sent this time
            data = data[nbytes:]

    def create_task(self, coro):
        self.ready.append(coro)

    def run_forever(self):
        while True:
            while not self.ready:
                events = self.selector.select()
                for key, _ in events:
                    self.selector.unregister(key.fileobj)
                    self.ready.append(key.data)

            while self.ready:
                # I need this task in another method, so I attach it here
                self.current_task = self.ready.popleft()
                try:
                    # drive the coroutine to the next yield
                    op, *args = self.current_task.send(None)
                except StopIteration:
                    pass
                else:
                    # call method on myself
                    # this methods will register sockets with selector
                    getattr(self, op)(*args)

    def read_wait(self, sock):
        # When this socket is ready, drive this task to the next yield
        self.selector.register(sock, EVENT_READ, self.current_task)

    def write_wait(self, sock):
        self.selector.register(sock, EVENT_WRITE, self.current_task)
Exemple #40
0
class TorSocketLoop(threading.Thread):
    def __init__(self, our_sock, send_func):
        super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno()))
        self._our_sock = our_sock
        self._send_func = send_func
        self._do_loop = True

        self._cntrl_r, self._cntrl_w = socket.socketpair()

        self._selector = DefaultSelector()
        self._selector.register(self._our_sock, EVENT_READ, self._do_recv)
        self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)

    def _do_recv(self, sock):
        data = sock.recv(1024)
        self._send_func(data)

    def _do_stop(self, sock):
        self._do_loop = False

    def _cleanup(self):
        self._selector.unregister(self._cntrl_r)
        self._cntrl_w.close()
        self._cntrl_r.close()
        self._selector.unregister(self._our_sock)
        self._our_sock.shutdown(socket.SHUT_WR)
        self._our_sock.close()
        self._selector.close()

    def stop(self):
        self._cntrl_w.send(b'\1')

    def run(self):
        logger.debug("Starting...")
        while self._do_loop:
            events = self._selector.select()
            for key, _ in events:
                callback = key.data
                callback(key.fileobj)

        self._cleanup()
        logger.debug("Stopped...")

    def append(self, data):
        self._our_sock.send(data)
#!/usr/env/bin python
import socket
import re

from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

selector = DefaultSelector()  # Linux system will use epoll()


class Task():
    def __init__(self, gen):
        self.gen = gen
        f = Future()
        f.set_result(None)
        self.step(f)

    def step(self, future):
        try:
            f = self.gen.send(future.result)
        except StopIteration:
            return
        f.add_done_callback(self.step)


class Future():
    def __init__(self):
 def __init__(self):
     self.selector = DefaultSelector()
     self.clients = {}
Exemple #43
0
class Peer(HandleableMixin, CommandableMixin, LanTopologyMixin,
           DefaultAuthenticatorMixin):
    """
    Attributes:
        server_info (PeerInfo): Peer's peer info.
        pkt_handlers (Dict[str, Handler]): All handlers which peer have ability
            to process.
        peer_pool (Dict[Tuple[str, int], PeerInfo]): All peers currently avai-
            lable in net.
    """
    @property
    def server_info(self):
        return self.__server_info

    @property
    def program_hash(self):
        return self.__program_hash

    @property
    def send_queue(self):
        # def packet_queue(self):
        return self.__packet_queue

    @property
    def connectlist(self) -> List:  # Remember to remove `List` import.
        return self.peer_pool.values()

    def __init__(
            self,
            host: Tuple[str, int],
            name: str,
            role: str,
            cert: Tuple[str, str],
            program_hash: str,
            ns: str,
            auto_register: bool = False,
            logger: "logging.Logger" = getLogger(__name__),
    ) -> None:
        """Init of PeerManager

        Args:
            host: Binding host.
            name: Peer's name in net.
            role: Peer's role in net.
            cert: Cert file's path.
            program_hash: Program self hash to send in packet.
            ns: Nameserver address for resolve DNS.
            logger: Logger for logging.
        """
        super().__init__()
        self.logger = getLogger(name)
        self.__auto_register = auto_register
        self.__selector = DefaultSelector()
        self.__packet_queue = {}

        self.__cert = cert
        self.__program_hash = program_hash
        self.__server_info = PeerInfo(host=host, name=name, role=role)
        self.__tcp_server = self.__bind_socket(cert=self.__cert)

        self.peer_pool = {}
        self.pkt_handlers = {}
        self.commands = {}

        self.logger.info("Program hash: {{{}...{}}}".format(
            self.__program_hash[:6], self.__program_hash[-6:]))
        self.dns_resolver = DNSResolver(ns="127.0.0.1" if ns is None else ns,
                                        role=role)
        self.monitor = Monitor(peer=self, logger=getLogger(name + ".MONITOR"))

        if self.__auto_register is False:
            self.logger.warning(
                ("auto_register parameter is set to False,\n You may need to r"
                 "egister them through _register_command & _register_handler m"
                 "ethod."))

    def _preregister_handler(self) -> None:
        self.topology_register_handler()
        installing_handlers = [MessageHandler(self)]
        for each in installing_handlers:
            self.register_handler(handler=each)

    def _preregister_command(self) -> None:
        installing_commands = [
            HelpCmd(self),
            JoinCmd(self),
            SendCmd(self),
            ListCmd(self),
            LeaveNetCmd(self),
        ]
        for each in installing_commands:
            self.register_command(command=each)

    def pend_packet(self, sock: "Socket", pkt: "Packet", **kwargs) -> None:
        """Pending pkt's raw_data to queue's with sepecific sock.
        Any exception when wrapping handler to packet whould cause this connec-
        tion been close and thread maintaining loop terminate.

        Args:
            sock: A Socket which wants to pend on its queue.
            pkt: A Packet ready to be pend.
            **kwargs: Any additional arguments needs by handler object.
        
        Raises:
            AssertionError:
                If given pkt variable is not in proper Packet type.
        """
        assert type(pkt) is Packet
        try:
            self.__packet_queue[sock].put_nowait(pkt)
        except Exception:
            self.logger.info(format_exc())

    def register_socket(self, sock: "Socket") -> None:
        """Register a new socket with packet queue & selector.
        Init a packet queue and put into dict for further handling of packets.
        And the given socket will be register in selector for IO process.

        Args:
            sock: A Socket object which wants to be register.
        """
        self.__packet_queue[sock] = Queue()
        self.__selector.register(sock, EVENT_READ | EVENT_WRITE,
                                 self.__on_handle)

    def unregister_socket(self, sock: "Socket") -> None:
        del self.__packet_queue[sock]
        self.__selector.unregister(sock)

    def _on_packet(self, sock: "Socket", pkt: "Packet",
                   handler: "Handler") -> None:
        """Method use to process passed packet to higher application layer.
        This method will call by AuthenticatorMixin when a packet is passed examination.
        
        This is 3rd layer to process packet to handler. This is last layer to application layer.
        """
        handler.on_recv(src=pkt.src, pkt=pkt, sock=sock)
        self.monitor.on_recv_pkt(addr=pkt.src, pkt=pkt, conn=sock)

    def new_tcp_long_conn(self, dst: Tuple[str, int]) -> "SSLSocket":
        """Create a ssl-wrapped TCP socket with given destination host

        Args:
            dst: Specified socket destination.

        Returns:
            A SSLSocket object which connected to destination host with
                non-blocking.

        Raises:
            AssertionError:
                If given dst variable is not in proper Tuple[str, int] type.
        """
        assert host_valid(dst) is True
        unwrap_socket = socket(AF_INET, SOCK_STREAM)
        sock = wrap_socket(unwrap_socket,
                           cert_reqs=CERT_REQUIRED,
                           ca_certs=self.__cert[0])

        sock.connect(dst)
        sock.setblocking(False)
        return sock

    def loop_start(self):
        self.logger.info(self.__server_info)
        self.__selector.register(self.__tcp_server, EVENT_READ,
                                 self.__on_accept)
        if self.__auto_register is True:
            self._preregister_handler()
            self._preregister_command()

        if self.monitor.is_start() is False:
            self.monitor.start()

        self.logger.info("Peer started.")

    def loop(self):
        return self._loop()

    def _loop(self):
        """Called inside infinite loop from outside inherited class.
        It's use to call the method which given event is triggered.
        """
        events = self.__selector.select(timeout=0)
        for key, mask in events:
            if callable(key.data):
                key.data(key.fileobj, mask)

    def loop_stop(self):
        for _, value in self.__packet_queue.items():
            if value.empty() is False:
                sleep(2)
                return self.loop_stop()

        self.__selector.unregister(self.__tcp_server)
        self.leave_net()

        self.monitor.stop()

    def loop_stop_post(self):
        self.__tcp_server.close()
        self.__selector.close()

    def __bind_socket(self, cert: Tuple[str, str]) -> "SSLSocket":
        unwrap_socket = socket(AF_INET, SOCK_STREAM)
        unwrap_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
        unwrap_socket.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
        unwrap_socket.bind(self.__server_info.host)
        unwrap_socket.listen(5)
        unwrap_socket.setblocking(False)

        self.logger.info("Peer prepared")
        self.logger.info(
            "This peer is running with certificate at path {}".format(cert[0]))
        self.logger.info("Please make sure other peers have same certicate.")
        return wrap_socket(unwrap_socket,
                           certfile=cert[0],
                           keyfile=cert[1],
                           server_side=True)

    def __on_accept(self, sock: "Socket", mask, **kwargs):
        """Call when a new socket is connection by waiter socket.
        This will accept all sockets from outside, but it doesn't mean every socket's
         packet be process by higher layer.
        This is the 1st layer to process sockets.
        """
        conn, _ = sock.accept()
        conn.setblocking(False)
        self.register_socket(sock=conn)

    def __on_handle(self, sock: "Socket", mask, **kwargs):
        """Decide whether send or recv."""
        if mask & EVENT_READ == EVENT_READ:
            self.__on_recv(sock=sock, mask=mask, **kwargs)
        if mask & EVENT_WRITE == EVENT_WRITE:
            self.__on_send(sock=sock, mask=mask, **kwargs)

    def __on_recv(self, sock: "Socket", mask, **kwargs):
        """Method use when recieve socket data.
        This is 2th layer to process Packets.
        """
        try:
            raw_data = sock.recv(4096)
            if raw_data == b"":
                return
            pkt = Packet.deserilize(raw_data=raw_data)
            return self._authenticate_packet(sock=sock, pkt=pkt)
        except SSLWantReadError:
            return
        except sock_error as sock_err:
            if sock_err.errno == ECONNRESET:
                peer_info = self.get_peer_info_by_conn(conn=sock)
                if peer_info is not None:
                    peer_info.status.update(status_type=StatusType.NO_RESP)
                    self.logger.warning("Peer {} Connection Reseted.".format(
                        peer_info.host))
            else:
                raise sock_err
        except Exception:
            self.logger.warning(str(self.server_info) + format_exc())

    def __on_send(self, sock: "Socket", mask, **kwargs):
        """Method use when sending data to socket."""
        q = self.__packet_queue[sock] if sock in self.__packet_queue else None
        while q is not None and q.empty() is False:
            try:
                pkt = q.get_nowait()
                handler = self.select_handler(pkt_type=pkt._type)
                handler.pre_send(pkt=pkt)
                data = Packet.serilize(obj=pkt)
                sock.send(data)
                handler.post_send(pkt=pkt, sock=sock)
            except sock_error as sock_err:
                if sock_err.errno == ECONNRESET:
                    q.put_nowait(pkt)
                    self.monitor.peer_status_update_by_host(
                        host=pkt.src, status_type=StatusType.NO_RESP)
                    self.logger.warning("Peer {} Connection Reseted.".format(
                        pkt.src))
                else:
                    raise sock_err
            except Exception:
                self.logger.warning(format_exc())
Exemple #44
0
class DBusConnection:
    def __init__(self, sock):
        self.sock = sock
        self.parser = Parser()
        self.outgoing_serial = count(start=1)
        self.selector = DefaultSelector()
        self.select_key = self.selector.register(sock, EVENT_READ)
        self._stop_r, self._stop_w = os.pipe()
        self.stop_key = self.selector.register(self._stop_r, EVENT_READ)
        self.send_lock = Lock()
        self.rcv_lock = Lock()
        self.unique_name = None

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
        return False

    def send(self, message: Message, serial=None):
        """Serialise and send a :class:`~.Message` object"""
        if serial is None:
            serial = next(self.outgoing_serial)
        data = message.serialise(serial=serial)
        with self.send_lock:
            self.sock.sendall(data)

    def receive(self, *, timeout=None) -> Message:
        """Return the next available message from the connection

        If the data is ready, this will return immediately, even if timeout<=0.
        Otherwise, it will wait for up to timeout seconds, or indefinitely if
        timeout is None. If no message comes in time, it raises TimeoutError.

        If the connection is closed from another thread, this will raise
        ReceiveStopped.
        """
        if timeout is not None:
            deadline = time.monotonic() + timeout
        else:
            deadline = None

        with self.rcv_lock:
            while True:
                msg = self.parser.get_next_message()
                if msg is not None:
                    return msg

                if deadline is not None:
                    timeout = deadline - time.monotonic()

                b = self._read_some_data(timeout)
                self.parser.add_data(b)

    def _read_some_data(self, timeout=None):
        # Wait for data or a signal on the stop pipe
        for key, ev in self.selector.select(timeout):
            if key == self.select_key:
                return unwrap_read(self.sock.recv(4096))
            elif key == self.stop_key:
                raise ReceiveStopped(
                    "DBus receive stopped from another thread")

        raise TimeoutError

    def interrupt(self):
        """Make any threads waiting for a message raise ReceiveStopped"""
        os.write(self._stop_w, b'a')

    def reset_interrupt(self):
        """Allow calls to .receive() again after .interrupt()

        To avoid race conditions, you should typically wait for threads to
        respond (e.g. by joining them) between interrupting and resetting.
        """
        # Clear any data on the stop pipe
        while (self.stop_key, EVENT_READ) in self.selector.select(timeout=0):
            os.read(self._stop_r, 1024)

    def close(self):
        """Close the connection"""
        self.interrupt()
        self.selector.close()
        self.sock.close()
Exemple #45
0
 def __init__(self, server_id, handler: Handler):
     self.select = DefaultSelector()
     self.server_id = server_id
     self.handler = handler
Exemple #46
0
class SimpleServer(object):
    """ This is a simple server receiving and sending utf-8 plain text lines
        over TCP/IP. It should use epoll or other efficient mechanisms if
        available to support a large number of clients without cpu hogging.

        All data sent to connections will be utf-8 encoded. All received lines
        from clients are expected to be utf-8 encoded.

        Lines longer than 10KB will be quietly dropped.
    """
    def __init__(self, port, callback):
        logging.basicConfig(level=logging.INFO)
        self.clients_kept_for_sending = []
        self.selector = DefaultSelector()
        self.callback = callback
        self.server_socket = socket.socket(family=socket.AF_INET6,
            type=socket.SOCK_STREAM)
        self.server_socket.bind(("::0", port))
        self.server_socket.listen(5)
        self.selector.register(self.server_socket, EVENT_READ, data=None)
        self.server_socket.setblocking(False)

    def close(self):
        self.server_socket.close()

    def _remove_socket(self, socket):
        try:
            self.selector.unregister(socket)
        except KeyError:
            pass

    def _handle_new_client(self, socket, addr):
        client = SimpleClient(socket, self, addr)
        self.selector.register(socket, EVENT_READ, data=client)
        self.callback(client)

    def _mark_client_for_sending(self, client, marked):
        if marked:
            self.selector.modify(client.socket, EVENT_READ | EVENT_WRITE,
                data=client)
        else:
            self.selector.modify(client.socket, EVENT_READ, data=client)

    def tick(self, timeout=0):
        """ This will check for any server events and process them.

            If timeout is greater than zero, it will wait at most that
            amount of time in seconds until something happens.
            (This is useful if you want to save some cpu time and you have
            nothing else to do)

            For debugging purposes, it returns the amount of events processed.
        """
        # get rid of old connections we kept around for sending:
        entry_removed = True
        while entry_removed:
            entry_removed = False
            for entry in self.clients_kept_for_sending:
                if entry[1] <= time.time() or len(entry[0].send_buffer) == 0:
                    try:
                        entry[0].close()
                    except:
                        pass
                    self.clients_kept_for_sending.remove(entry)
                    entry_removed = True

        # wait for socket events:
        events = self.selector.select(timeout=timeout)
        events_processed = 0
        for key, mask in events:
            if mask & EVENT_READ != 0:
                events_processed = events_processed + 1
                # handle accept for server socket:
                if key.data == None:
                    conn, addr = self.server_socket.accept()
                    self._handle_new_client(conn, addr)
                    continue
                # handle client socket receive:
                key.data._do_recv()
                continue
            if mask & EVENT_WRITE != 0:
                events_processed = events_processed + 1
                # handle client socket send:
                key.data._do_outstanding_send()
                continue
            logging.debugging("unknown socket event happening, bug?")
        return events_processed
Exemple #47
0
import argparse
from selectors import DefaultSelector, EVENT_WRITE
from socket import socket

# 网络连接的例子(socket和selectors结合)


def connected():
    selector.unregister(sock.fileno())
    print('connected!')


url = "http://www.baicu.com"
sock = socket()
sock.setblocking(False)
sock.connect(('xkcd.com', 80))
selector = DefaultSelector()
selector.register(sock.fileno(), connected())
 def __init__(self):
     self.sel = DefaultSelector()
     self.queue = deque()
Exemple #49
0
 def selector():
     sel = DefaultSelector()
     sel.register(proc.stdout.channel, EVENT_READ)
     while True:
         for key, mask in sel.select():
             yield
Exemple #50
0
 def __init__(self):
     self.ready = collections.deque()
     self.selector = DefaultSelector()
Exemple #51
0
import socket
from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ


sock = socket.socket()
sock.setblocking(False)

selector = DefaultSelector()

urls_todo = set(['/'])
seen_urls = set(['/'])


class Fetcher:
    def __init__(self, url):
        self.response = b''
        self.url = url
        self.sock = None

    def fetch(self):
        # This method fetches the url
        self.sock = socket.socket()
        self.sock.setblocking(False)
        try:
            self.sock.connect(('xkcd.com'), 80)
        except BlockingIOError:
            pass

        selector.register(self.sock.fileno(),
                          EVENT_WRITE,
                          self.connected)
# coding=utf-8
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE

epoller = DefaultSelector()
stopped = False
urls_todo = {'/' + str(i) for i in range(10)}


class Future:
    def __init__(self):
        self.result = None
        self._callbacks = []

    def add_done_callback(self, fn):
        self._callbacks.append(fn)

    def set_result(self, result):
        self.result = result
        for fn in self._callbacks:
            fn(self)

    # 改造为一个可迭代对象
    def __iter__(self):
        yield self
        return self.result


# 抽象sock连接功能
def connect(sock, address):
    f = Future()
Exemple #53
0
class LibInput(object):
    """A base/factory class for libinput context.

	Context is used to manage devices and get events.
	"""

    _libc = CDLL('libc.so.6')
    _libc.vsprintf.argtypes = (c_char_p, c_char_p, c_void_p)
    _libc.vsprintf.restype = c_int
    _libudev = CDLL('libudev.so.1')
    _libudev.udev_new.argtypes = None
    _libudev.udev_new.restype = c_void_p
    _libudev.udev_unref.argtypes = (c_void_p, )
    _libudev.udev_unref.restype = None
    # libinput is not available on RTD so we prevent failing immediately
    # when it's not available. Trying to instantiate the class will still
    # throw an exception.
    try:
        _libinput = CDLL('libinput.so.10')
        _libinput.libinput_udev_create_context.argtypes = (POINTER(
            Interface.Interface), c_void_p, c_void_p)
        _libinput.libinput_udev_create_context.restype = c_void_p
        _libinput.libinput_udev_assign_seat.argtypes = (c_void_p, c_char_p)
        _libinput.libinput_udev_assign_seat.restype = c_int
        _libinput.libinput_path_create_context.argtypes = (POINTER(
            Interface.Interface), c_void_p)
        _libinput.libinput_path_create_context.restype = c_void_p
        _libinput.libinput_unref.argtypes = (c_void_p, )
        _libinput.libinput_unref.restype = c_void_p
        _libinput.libinput_log_set_handler.argtypes = (c_void_p, c_void_p)
        _libinput.libinput_log_set_handler.restype = None
        _libinput.libinput_log_set_priority.argtypes = (c_void_p, LogPriority)
        _libinput.libinput_log_set_priority.restype = None
        _libinput.libinput_get_fd.argtypes = (c_void_p, )
        _libinput.libinput_get_fd.restype = c_int
        _libinput.libinput_suspend.argtypes = (c_void_p, )
        _libinput.libinput_suspend.restype = None
        _libinput.libinput_resume.argtypes = (c_void_p, )
        _libinput.libinput_resume.restype = c_int
        _libinput.libinput_path_add_device.argtypes = (c_void_p, c_char_p)
        _libinput.libinput_path_add_device.restype = c_void_p
        _libinput.libinput_path_remove_device.argtypes = (c_void_p, )
        _libinput.libinput_path_remove_device.restype = None
        _libinput.libinput_dispatch.argtypes = (c_void_p, )
        _libinput.libinput_dispatch.restype = c_int
        _libinput.libinput_get_event.argtypes = (c_void_p, )
        _libinput.libinput_get_event.restype = c_void_p
        _libinput.libinput_event_get_type.argtypes = (c_void_p, )
        _libinput.libinput_event_get_type.restype = EventType

        _libinput.libinput_next_event_type.argtypes = (c_void_p, )
        _libinput.libinput_next_event_type.restype = c_int
    except OSError:
        pass

    def __new__(cls, context_type=ContextType.PATH, debug=False):

        if context_type == ContextType.PATH:
            return LibInputPath()
        elif context_type == ContextType.UDEV:
            return LibInputUdev()
        else:
            raise TypeError('Unsupported context type')

    def __init__(self, context_type=ContextType.PATH, debug=False):
        """Initialize context.

		Args:
			context_type (~libinput.constant.ContextType): If
				:attr:`~libinput.constant.ContextType.UDEV` devices are
				added/removed from udev seat. If
				:attr:`~libinput.constant.ContextType.PATH` devices have to be
				added/removed manually.
			debug (bool): If false, only errors are printed.
		"""

        self._selector = DefaultSelector()
        self._interface = Interface()
        if context_type == ContextType.UDEV:
            self._udev = self._libudev.udev_new()
            self._li = self._libinput.libinput_udev_create_context(
                byref(self._interface), None, self._udev)
        elif context_type == ContextType.PATH:
            self._li = self._libinput.libinput_path_create_context(
                byref(self._interface), None)
        self._log_handler = lambda pr, strn: print(pr.name, ': ', strn)
        self._set_default_log_handler()
        if debug:
            self._libinput.libinput_log_set_priority(self._li,
                                                     LogPriority.DEBUG)
        self._selector.register(self._libinput.libinput_get_fd(self._li),
                                EVENT_READ)

    def __del__(self):

        while self._libinput.libinput_unref(self._li):
            pass

    def _set_default_log_handler(self):
        def default_log_handler(li, priority, fmt, args):

            string = create_string_buffer(2048)
            self._libc.vsprintf(string, fmt, args)
            self._log_handler(LogPriority(priority), string.value.decode())

        CMPFUNC = CFUNCTYPE(None, c_void_p, c_int, c_char_p, c_void_p)
        self._default_log_handler = CMPFUNC(default_log_handler)
        self._libinput.libinput_log_set_handler(self._li,
                                                self._default_log_handler)

    @property
    def log_handler(self):
        """Callable that handles error/info/debug messages.

		Args:
			priority (~libinput.constant.LogPriority): Message priority.
			message (str): The message.
		Default handler prints messages to stdout.
		"""

        return self._log_handler

    @log_handler.setter
    def log_handler(self, handler):

        self._log_handler = handler

    def suspend(self):
        """Suspend monitoring for new devices and close existing devices.

		This all but terminates libinput but does keep the context valid to be
		resumed with :meth:`resume`.
		"""

        self._libinput.libinput_suspend(self._li)

    def resume(self):
        """Resume a suspended libinput context.

		This re-enables device monitoring and adds existing devices.
		"""

        rc = self._libinput.libinput_resume(self._li)
        assert rc == 0, 'Failed to resume current context'

    @property
    def events(self):
        """Yield events from the internal libinput's queue.

		Yields device events that are subclasses of
		:class:`~libinput.event.Event`.

		Yields:
			:class:`~libinput.event.Event`: Device event.
		"""

        while True:
            events = self._selector.select()
            for nevent in range(len(events) + 1):
                self._libinput.libinput_dispatch(self._li)
                hevent = self._libinput.libinput_get_event(self._li)
                if hevent:
                    type_ = self._libinput.libinput_event_get_type(hevent)
                    self._libinput.libinput_dispatch(self._li)
                    if type_.is_pointer():
                        yield PointerEvent(hevent, self._libinput)
                    elif type_.is_keyboard():
                        yield KeyboardEvent(hevent, self._libinput)
                    elif type_.is_touch():
                        yield TouchEvent(hevent, self._libinput)
                    elif type_.is_gesture():
                        yield GestureEvent(hevent, self._libinput)
                    elif type_.is_tablet_tool():
                        yield TabletToolEvent(hevent, self._libinput)
                    elif type_.is_tablet_pad():
                        yield TabletPadEvent(hevent, self._libinput)
                    elif type_.is_switch():
                        yield SwitchEvent(hevent, self._libinput)
                    elif type_.is_device():
                        yield DeviceNotifyEvent(hevent, self._libinput)

    def next_event_type(self):
        """Return the type of the next event in the internal queue.

		This method does not pop the event off the queue and the next call
		to :attr:`events` returns that event.

		Returns:
			~libinput.constant.EventType: The event type of the next available
			event or :obj:`None` if no event is available.
		"""

        type_ = self._libinput.libinput_next_event_type(self._li)
        if type_ == 0:
            return None
        else:
            return EventType(type_)
Exemple #54
0
class Loop:
    def __init__(self):
        self._tasks = deque()
        self._sleeping = []
        self._stop = False
        self._selector = DefaultSelector()

    def create_task(self, coroutine):
        if isinstance(coroutine, Task):
            return coroutine

        task = Task(coroutine)
        self._tasks.append(task)
        return task

    def remove(self, task):
        self.tasks.remove(task)

    def _stop_loop_cb(self, task):
        self._stop = True

    def _wakeup_sleeping_tasks(self):
        now = time.monotonic()

        while self._sleeping and self._sleeping[0][0] <= now:
            _, task = heapq.heappop(self._sleeping)
            self._tasks.append(task)

    def io_poll(self):
        if self._sleeping:
            sleep_time = self._sleeping[0][0] - time.monotonic()
        else:
            sleep_time = None

        events = self._selector.select(sleep_time)
        for key, mask in events:
            task = key.data
            self._selector.unregister(key.fileobj)
            self._tasks.append(task)

        # Wake up time!
        self._wakeup_sleeping_tasks()

    def run_until_complete(self, coroutine):
        task = self.create_task(coroutine)
        task.add_done_callback(self._stop_loop_cb)
        self.run_forever()
        task.remove_done_callback(self._stop_loop_cb)
        return task.result

    def run_forever(self):
        self._stop = False  # Just in case we are restarted

        while not self._stop:
            while not self._tasks:
                self.io_poll()

                # Let's see if there any task who wants to wake up
                self._wakeup_sleeping_tasks()

            current = self._tasks.popleft()
            try:
                # We assume that each Task will yield some method name and
                # an argument. Based on that, we decide what to do.
                method_name, arg = next(current)
            except StopIteration as e:
                current.result = e.value
            except Exception as e:
                current.exception = e
                print(f'Warning: Task {current.id} raised {e!r}')
            else:
                method = getattr(self, method_name, None)
                assert method, f'unknown method name {method_name}'
                method(arg, current)
            finally:
                if current.done:
                    for callback in current.done_callbacks:
                        callback(current)

    # These are the methods which can be "invoked" by Tasks by yielding their
    # name, together with an argument.
    def sleep_task(self, seconds, task):
        # We simply add the task to the self._sleeping priority queue.
        # The use of a priority queue here is very convenient: we push
        # (time to wake up, task) into the queue. When we get/pop an item from
        # the queue, we automatically get the fisrt task to wake up together
        # with its wakeup time.
        heapq.heappush(self._sleeping, (time.monotonic() + seconds, task))

    def read_wait(self, resource, task):
        self._selector.register(resource, EVENT_READ, task)

    def write_wait(self, resource, task):
        self._selector.register(resource, EVENT_WRITE, task)

    # Socket stuff
    def sock_recv(self, sock, maxsize):
        yield 'read_wait', sock
        return sock.recv(maxsize)

    def sock_sendall(self, sock, data):
        while data:
            try:
                n = sock.send(data)
                data = data[n:]
            except BlockingIOError:
                yield 'write_wait', sock

    def sock_accept(self, sock):
        yield 'read_wait', sock
        return sock.accept()

    def sock_connect(self, sock, address):
        yield 'write_wait', sock
        return sock.connect(address)
Exemple #55
0
class EventLoop:
    def __init__(self):
        self.tasks = deque()
        self.selector = DefaultSelector()

    def run_until_complete(self, task):
        self.tasks.append(task)
        self.run()

    def pause(self):
        return "pause", None

    def schedule(self, target):
        return "schedule", target

    def sock_accept(self, sock):
        yield ("read", sock)
        return sock.accept()

    def sock_recv(self, sock):
        yield ("read", sock)
        return sock.recv(1024)

    def sock_sendall(self, sock, data):
        yield ("write", sock)
        return sock.sendall(data)

    def start_server(self, handler, host, port, backlog=0):
        handler = partial(handler, self)
        with socket.socket() as sock:
            sock.bind((host, port))
            sock.listen(backlog)
            print("Listening on {}:{}".format(host, port))
            while True:
                conn, addr = yield from self.sock_accept(sock)
                print("Accepted client from", addr)
                yield self.schedule(handler(conn))

    def run(self):
        while self.tasks or self.selector.get_map():
            for _ in range(len(self.tasks)):
                try:
                    task = self.tasks.popleft()
                    tag, value = next(task)

                    if tag == "schedule":
                        self.tasks.append(value)
                        self.tasks.append(task)
                    elif tag == "read":
                        self.selector.register(value, EVENT_READ, data=task)
                    elif tag == "write":
                        self.selector.register(value, EVENT_WRITE, data=task)
                    elif tag == "pause":
                        self.tasks.append(task)
                    else:
                        raise ValueError("Incorrect tag")
                except StopIteration:
                    continue

            if self.selector.get_map():
                for key, event in self.selector.select():
                    if event & EVENT_READ or event & EVENT_WRITE:
                        self.tasks.append(key.data)
                    self.selector.unregister(key.fileobj)
Exemple #56
0
 def __init__(self):
     self._tasks = deque()
     self._sleeping = []
     self._stop = False
     self._selector = DefaultSelector()
Exemple #57
0
 def __init__(self):
     self.ready = deque()
     self.selector = DefaultSelector()
#python3.4 or higher is required in order to use selectors
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
from datetime import datetime

sel = DefaultSelector()
request = "GET / HTTP/1.0\r\n\r\n"

request_callback = lambda sock: sock.send(request.encode())
response_callback = lambda sock: \
        print((sock.recv(1000)).decode().split("\n")[-1], 'finish at %s' % str(datetime.now())[:-7])

def get():
    sock = socket.socket()
    sock.setblocking(False)
    try:
        sock.connect(('localhost', 3000))
    except:
        pass
    sel.register(sock, EVENT_WRITE, lambda: request_callback(sock))

get_count = 4
print("starts at %s" % str(datetime.now())[:-7])
for i in range(get_count):
    get()

while True:
    events = sel.select()
    for key, _ in events:
        callback = key.data
        event = key.events
 def __init__(self):
     self.ready = deque()
     self.selector = DefaultSelector()
     self.futures = {}
Exemple #60
0
def distribute(cmd, max_bytes, max_procs, chunk_size, round_robin, verbose):
    """
    Blocking function that manages all the delegation of chunks from
    stdin to subprocesses, spawning of subprocess, and collation of
    stdout and stderr from each subprocess.

    Broadly speaking, gatling reads chunks of data from stdin and
    disperses those among subprocesses to achieve parallel execution.

    Stdin is read in chunks of chunk_size bytes and truncated to the
    last newline, keeping the remainder to be prepended on the next
    chunk. Multiple chunks without newline is allowed, nothing is
    passed on until a newline is found or stdin closes.

    The stdout and stderr from each child is read in a similar
    fashion. Whenever a newline is found the output is written onto
    stdout or stderr, respectively. This collation preserves the
    format of the output, but only weakly adheres to the chronology.
    The output from gatling is the result of the subprocesses with a
    line-by-line integrity preserved with no guarantee that the order
    is exactly maintained.

    There are two different behaviors for subprocess spawning,
    manifold or gatling:

    * In manifold-mode each new chunk read from stdin spawns a new
      subprocess until max_procs is reached and then each of the
      subprocesses are fed a chunk in round robin fashion. This is an
      excellent model for programs that do not tax an external
      resource and programs that can act on stdin as soon as it is
      available.

    * In gatling-mode chunks are fed to a single subprocess until that
      processes' max_bytes is reached, the subprocess' stdin is closed
      and on the next chunk a new subprocess is spawned and fed until
      its max_bytes is reached and so on until max_procs is reached.
      If max_procs is reached gatling is blocked until a subprocess
      finishes. This mode works well for programs that connect to an
      external service or programs that don't start processing stdin
      until it is closed.

    cmd (list): The full command line for spawning subprocesses.

    max_bytes (int): Maximum bytes to pass to each subprocess before
        closing the subprocess' stdin. Increase this value if
        subprocesses do not have memory management problems from large
        input and if subprocesses process stdin as stream. Decrease
        this value if programs can not handle a large input set on
        stdin or if programs do not start processing until stdin is
        closed.

    max_procs (int): Maximum number of simultaneos subprocesses.
        Increase this number if the subprocess is largely CPU bound,
        decrease it to match the hardware if the subprocesses are IO
        bound.

    chunk_size (int): Stdin content streamed to gatling is consumed in
        chunks of this size (bytes) for efficiency reasons. Originally
        it was line by line, but that was too slow to keep
        subprocesses fed continuously. To force line-by-line behavior,
        set chunk to a size always less than the length of an input
        line (worst case 1, but try to keep it as high as possible).
        Experiment with this number to maximize pipeline throughput.

    round_robin (bool): True is manifold, false is gatling, see above
        for details.
    """

    from sys import stdin, stdout, stderr
    from subprocess import Popen, PIPE
    from time import sleep, time
    from selectors import DefaultSelector, EVENT_READ
    from threading import Thread
    from fcntl import fcntl, F_SETFL, F_GETFL
    from os import O_NONBLOCK

    def sink(selector, not_done):
        n = 0
        while len(not_done) > 1 or list(selector.get_map()):
            for (fileobj, _, _, p), _ in selector.select():
                chunk = fileobj.read(4096)
                if chunk:
                    i = chunk.rfind('\n') + 1
                    if i:
                        n += fileobj._trg.write(fileobj._buf)
                        n += fileobj._trg.write(chunk[:i])
                        fileobj._buf = chunk[i:]
                    else:
                        fileobj._buf += chunk
                else:
                    if p.returncode is not None:
                        n += fileobj._trg.write(fileobj._buf)
                        selector.unregister(fileobj)
            not_done[0] = n

    my_name = 'manifold' if round_robin else 'gatling'

    selector = DefaultSelector()
    res = []
    p_filled = []  # Child processes that have had their maximum input supplied
    # (p_open) Child processes that can take more input
    # In non-round-robin mode, there will only be one such child,
    # which will continually be popped and re-appended.
    # In round-robin mode, this list is rotated each time there is
    # a buffer to be written to a child.
    p_open = []
    b_in = 0
    buf = ''
    not_done = [0, 1]
    sel_t = None
    t0 = time()
    try:
        for chunk in iter(lambda: stdin.read(chunk_size), ''):
            b_in += len(chunk)
            i = chunk.rfind('\n') + 1
            if i:
                p = None
                if round_robin:
                    if len(p_open) + len(p_filled) == max_procs:
                        p = p_open.pop(0)
                else:
                    if p_open:
                        p = p_open.pop(0)
                if not p:
                    if verbose:
                        running = len(p_filled) + len(p_open)
                        print(
                            f"# {my_name} STARTED A PROCESS (1 + {running} + {len(res)}):",
                            *cmd,
                            file=sys.stderr)
                    p = Popen(cmd,
                              encoding=stdout.encoding,
                              stdin=PIPE,
                              stdout=PIPE,
                              stderr=PIPE)
                    p._n = 0
                    p._t0 = time()
                    for fo, trg in [(p.stdout, stdout), (p.stderr, stderr)]:
                        fo._buf = ''
                        fo._trg = trg
                        fcntl(fo, F_SETFL, fcntl(fo, F_GETFL) | O_NONBLOCK)
                        selector.register(fo, EVENT_READ, p)
                    if not sel_t:
                        sel_t = Thread(target=sink, args=(selector, not_done))
                        sel_t.daemon = True
                        sel_t.start()

                p._n += p.stdin.write(buf)
                p._n += p.stdin.write(chunk[:i])
                buf = chunk[i:]
                if p._n >= max_bytes:
                    t1 = time()
                    p._t1 = t1
                    td = (t1 - t0) * 1024
                    ptd = (t1 - p._t0) * 1024
                    p.stdin.close()
                    p_filled.append(p)
                    if verbose and ptd and td:
                        running = len(p_filled) + len(p_open)
                        print(
                            f"# {my_name} PROCESS LIMIT {p._n:,}/{max_bytes:,}",
                            f"({p._n/ptd:.2f} kb/s).",
                            f"INPUT: {b_in:,} ({b_in/td:.2f} kb/s) OUTPUT: {not_done[0]:,}.",
                            f"PROCESSES: {running}/{running+len(res)}",
                            file=sys.stderr)
                    while len(p_filled) == max_procs:
                        done = [d for d in p_filled if d.poll() is not None]
                        if done and verbose:
                            print(f"# {my_name} CLOSED {len(done)} PROCESSES",
                                  file=sys.stderr)
                        for d in done:
                            if verbose and p._t0 != t1:
                                print(
                                    f"# {my_name} CLOSED PROCESS INPUT: {p._n:,} TIME:",
                                    f"{t1-p._t0:.1f}/{t1-p._t1:.1f}",
                                    f"KB/S: {p._n/(t1-p._t0)/1024:.2f}",
                                    file=sys.stderr)
                            p_filled.remove(d)
                            res.append(d.returncode)
                        if not done:
                            sleep(0.5)
                else:
                    p_open.append(p)
            else:
                buf += chunk
        for p in p_open:
            p.stdin.write(buf)
            buf = ''
            p.stdin.close()
            p_filled.append(p)
    except (KeyboardInterrupt, SystemExit):
        not_done.pop()
        for p in p_open:
            p.stdin.close()
            p.kill()
        for d in p_filled:
            d.kill()
        raise
    while p_filled:
        res.append(p_filled.pop(0).wait())

    if sel_t:
        not_done.pop()
        sel_t.join()
    selector.close()
    return res