def __init__(self): self.clients = {} self.selector = DefaultSelector() self.lock = RLock() self.validationThread = Thread(target=self.validate) self.validationThread.start() self.logger = logging.getLogger(__name__)
def __init__(self, address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): self.address = address self.timeout = timeout self.source_address = source_address self._error = None self._selector = DefaultSelector() self._sockets = [] self._start_time = None
def main(): configs = cu.get_config(__file__) cu.set_main_log(__file__) logger = lu.get_logger(__file__) selector = DefaultSelector() connector = Connector(configs, selector) logger.info('Starting the proxy') logger.info('logger name: {}'.format(cu.MAIN_LOG)) Driver(connector.forward()) loop(selector)
# coding=utf8 import socket from selectors2 import DefaultSelector, EVENT_READ, EVENT_WRITE import time url = 'xkcd.com' selector = DefaultSelector() stopped = False res = [] def log_time(func): def wrapper(*args, **kw): start_time = time.time() func(*args, **kw) print(time.time() - start_time) return wrapper class Crawler(object): def __init__(self, url): self.url = url self.sock = None self.response = b'' def fetch(self): self.sock = socket.socket() self.sock.setblocking(False) try:
class Idlepool(metaclass=Singleton): def __init__(self): self.clients = {} self.selector = DefaultSelector() self.lock = RLock() self.validationThread = Thread(target=self.validate) self.validationThread.start() self.logger = logging.getLogger(__name__) def validate(self): while True: hasClients = False with self.lock: hasClients = len(self.clients) > 0 if not hasClients: time.sleep(5) continue events = self.selector.select(timeout=15) clientsToNofify = [] for key, _ in events: clientId = key.data clientsToNofify.append(clientId) for clientId in clientsToNofify: self.notifyClient(clientId) def notifyClient(self, clientId): with self.lock: if clientId in self.clients: dict = self.clients[clientId] event = dict['event'] event.set() def registerClient(self, clientId, client, event): if not clientId or not client or not event: return with self.lock: if clientId in self.clients: return try: self.clients[clientId] = { 'client' : client, 'event' : event } self.selector.register(client._imap.sock, EVENT_READ, clientId) except Exception as e: self.logger.exception('[%s] Could not register client with exception: %s', clientId, e) def unregisterClient(self, clientId): if not clientId: return with self.lock: if clientId in self.clients: dict = self.clients[clientId] client = dict['client'] try: self.selector.unregister(client._imap.sock) del self.clients[clientId] except Exception as e: self.logger.exception('[%s] Could not unregister client with exception: %s', clientId, e)
class _RFC6555ConnectionManager(object): def __init__(self, address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): self.address = address self.timeout = timeout self.source_address = source_address self._error = None self._selector = DefaultSelector() self._sockets = [] self._start_time = None def create_connection(self): self._start_time = perf_counter() host, port = self.address addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) ret = self._connect_with_cached_family(addr_info) # If it's a list, then these are the remaining values to try. if isinstance(ret, list): addr_info = ret else: cache.add_entry(self.address, ret.family) return ret # If we don't get any results back then just skip to the end. if not addr_info: raise socket.error('getaddrinfo returns an empty list') sock = self._attempt_connect_with_addr_info(addr_info) if sock: cache.add_entry(self.address, sock.family) return sock elif self._error: raise self._error else: raise socket.timeout() def _attempt_connect_with_addr_info(self, addr_info): sock = None try: for family, socktype, proto, _, sockaddr in addr_info: self._create_socket(family, socktype, proto, sockaddr) sock = self._wait_for_connection(False) if sock: break if sock is None: sock = self._wait_for_connection(True) finally: self._remove_all_sockets() return sock def _connect_with_cached_family(self, addr_info): family = cache.get_entry(self.address) if family is None: return addr_info is_family = [] not_family = [] for value in addr_info: if value[0] == family: is_family.append(value) else: not_family.append(value) sock = self._attempt_connect_with_addr_info(is_family) if sock is not None: return sock return not_family def _create_socket(self, family, socktype, proto, sockaddr): sock = None try: sock = socket.socket(family, socktype, proto) # If we're using the 'default' socket timeout we have # to set it to a real value here as this is the earliest # opportunity to without pre-allocating a socket just for # this purpose. if self.timeout is socket._GLOBAL_DEFAULT_TIMEOUT: self.timeout = sock.gettimeout() if self.source_address: sock.bind(self.source_address) # Make the socket non-blocking so we can use our selector. sock.settimeout(0.0) if self._is_acceptable_errno(sock.connect_ex(sockaddr)): self._selector.register(sock, EVENT_WRITE) self._sockets.append(sock) except _SOCKET_ERRORS as e: self._error = e if sock is not None: _RFC6555ConnectionManager._close_socket(sock) def _wait_for_connection(self, last_wait): self._remove_all_errored_sockets() # This is a safe-guard to make sure sock.gettimeout() is called in the # case that the default socket timeout is used. If there are no # sockets then we may not have called sock.gettimeout() yet. if not self._sockets: return None # If this is the last time we're waiting for connections # then we should wait until we should raise a timeout # error, otherwise we should only wait >0.2 seconds as # recommended by RFC 6555. if last_wait: if self.timeout is None: select_timeout = None else: select_timeout = self._get_remaining_time() else: select_timeout = self._get_select_time() # Wait for any socket to become writable as a sign of being connected. for key, _ in self._selector.select(select_timeout): sock = key.fileobj if not self._is_socket_errored(sock): # Restore the old proper timeout of the socket. sock.settimeout(self.timeout) # Remove it from this list to exempt the socket from cleanup. self._sockets.remove(sock) self._selector.unregister(sock) return sock return None def _get_remaining_time(self): if self.timeout is None: return None return max(self.timeout - (perf_counter() - self._start_time), 0.0) def _get_select_time(self): if self.timeout is None: return 0.2 return min(0.2, self._get_remaining_time()) def _remove_all_errored_sockets(self): socks = [] for sock in self._sockets: if self._is_socket_errored(sock): socks.append(sock) for sock in socks: self._selector.unregister(sock) self._sockets.remove(sock) _RFC6555ConnectionManager._close_socket(sock) @staticmethod def _close_socket(sock): try: sock.close() except _SOCKET_ERRORS: pass def _is_acceptable_errno(self, errno): if errno == 0 or errno in _ASYNC_ERRNOS: return True self._error = socket.error() self._error.errno = errno return False def _is_socket_errored(self, sock): errno = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) return not self._is_acceptable_errno(errno) def _remove_all_sockets(self): for sock in self._sockets: self._selector.unregister(sock) _RFC6555ConnectionManager._close_socket(sock) self._sockets = []