def __init__(self):
self.clients = {}
self.selector = DefaultSelector()
self.lock = RLock()
self.validationThread = Thread(target=self.validate)
self.validationThread.start()
self.logger = logging.getLogger(__name__)
def __init__(self,
address,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
self.address = address
self.timeout = timeout
self.source_address = source_address
self._error = None
self._selector = DefaultSelector()
self._sockets = []
self._start_time = None
def main():
configs = cu.get_config(__file__)
cu.set_main_log(__file__)
logger = lu.get_logger(__file__)
selector = DefaultSelector()
connector = Connector(configs, selector)
logger.info('Starting the proxy')
logger.info('logger name: {}'.format(cu.MAIN_LOG))
Driver(connector.forward())
loop(selector)
# coding=utf8
import socket
from selectors2 import DefaultSelector, EVENT_READ, EVENT_WRITE
import time
url = 'xkcd.com'
selector = DefaultSelector()
stopped = False
res = []
def log_time(func):
def wrapper(*args, **kw):
start_time = time.time()
func(*args, **kw)
print(time.time() - start_time)
return wrapper
class Crawler(object):
def __init__(self, url):
self.url = url
self.sock = None
self.response = b''
def fetch(self):
self.sock = socket.socket()
self.sock.setblocking(False)
try:
class Idlepool(metaclass=Singleton):
def __init__(self):
self.clients = {}
self.selector = DefaultSelector()
self.lock = RLock()
self.validationThread = Thread(target=self.validate)
self.validationThread.start()
self.logger = logging.getLogger(__name__)
def validate(self):
while True:
hasClients = False
with self.lock:
hasClients = len(self.clients) > 0
if not hasClients:
time.sleep(5)
continue
events = self.selector.select(timeout=15)
clientsToNofify = []
for key, _ in events:
clientId = key.data
clientsToNofify.append(clientId)
for clientId in clientsToNofify:
self.notifyClient(clientId)
def notifyClient(self, clientId):
with self.lock:
if clientId in self.clients:
dict = self.clients[clientId]
event = dict['event']
event.set()
def registerClient(self, clientId, client, event):
if not clientId or not client or not event:
return
with self.lock:
if clientId in self.clients:
return
try:
self.clients[clientId] = { 'client' : client, 'event' : event }
self.selector.register(client._imap.sock, EVENT_READ, clientId)
except Exception as e:
self.logger.exception('[%s] Could not register client with exception: %s', clientId, e)
def unregisterClient(self, clientId):
if not clientId:
return
with self.lock:
if clientId in self.clients:
dict = self.clients[clientId]
client = dict['client']
try:
self.selector.unregister(client._imap.sock)
del self.clients[clientId]
except Exception as e:
self.logger.exception('[%s] Could not unregister client with exception: %s', clientId, e)
class _RFC6555ConnectionManager(object):
def __init__(self,
address,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
self.address = address
self.timeout = timeout
self.source_address = source_address
self._error = None
self._selector = DefaultSelector()
self._sockets = []
self._start_time = None
def create_connection(self):
self._start_time = perf_counter()
host, port = self.address
addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
ret = self._connect_with_cached_family(addr_info)
# If it's a list, then these are the remaining values to try.
if isinstance(ret, list):
addr_info = ret
else:
cache.add_entry(self.address, ret.family)
return ret
# If we don't get any results back then just skip to the end.
if not addr_info:
raise socket.error('getaddrinfo returns an empty list')
sock = self._attempt_connect_with_addr_info(addr_info)
if sock:
cache.add_entry(self.address, sock.family)
return sock
elif self._error:
raise self._error
else:
raise socket.timeout()
def _attempt_connect_with_addr_info(self, addr_info):
sock = None
try:
for family, socktype, proto, _, sockaddr in addr_info:
self._create_socket(family, socktype, proto, sockaddr)
sock = self._wait_for_connection(False)
if sock:
break
if sock is None:
sock = self._wait_for_connection(True)
finally:
self._remove_all_sockets()
return sock
def _connect_with_cached_family(self, addr_info):
family = cache.get_entry(self.address)
if family is None:
return addr_info
is_family = []
not_family = []
for value in addr_info:
if value[0] == family:
is_family.append(value)
else:
not_family.append(value)
sock = self._attempt_connect_with_addr_info(is_family)
if sock is not None:
return sock
return not_family
def _create_socket(self, family, socktype, proto, sockaddr):
sock = None
try:
sock = socket.socket(family, socktype, proto)
# If we're using the 'default' socket timeout we have
# to set it to a real value here as this is the earliest
# opportunity to without pre-allocating a socket just for
# this purpose.
if self.timeout is socket._GLOBAL_DEFAULT_TIMEOUT:
self.timeout = sock.gettimeout()
if self.source_address:
sock.bind(self.source_address)
# Make the socket non-blocking so we can use our selector.
sock.settimeout(0.0)
if self._is_acceptable_errno(sock.connect_ex(sockaddr)):
self._selector.register(sock, EVENT_WRITE)
self._sockets.append(sock)
except _SOCKET_ERRORS as e:
self._error = e
if sock is not None:
_RFC6555ConnectionManager._close_socket(sock)
def _wait_for_connection(self, last_wait):
self._remove_all_errored_sockets()
# This is a safe-guard to make sure sock.gettimeout() is called in the
# case that the default socket timeout is used. If there are no
# sockets then we may not have called sock.gettimeout() yet.
if not self._sockets:
return None
# If this is the last time we're waiting for connections
# then we should wait until we should raise a timeout
# error, otherwise we should only wait >0.2 seconds as
# recommended by RFC 6555.
if last_wait:
if self.timeout is None:
select_timeout = None
else:
select_timeout = self._get_remaining_time()
else:
select_timeout = self._get_select_time()
# Wait for any socket to become writable as a sign of being connected.
for key, _ in self._selector.select(select_timeout):
sock = key.fileobj
if not self._is_socket_errored(sock):
# Restore the old proper timeout of the socket.
sock.settimeout(self.timeout)
# Remove it from this list to exempt the socket from cleanup.
self._sockets.remove(sock)
self._selector.unregister(sock)
return sock
return None
def _get_remaining_time(self):
if self.timeout is None:
return None
return max(self.timeout - (perf_counter() - self._start_time), 0.0)
def _get_select_time(self):
if self.timeout is None:
return 0.2
return min(0.2, self._get_remaining_time())
def _remove_all_errored_sockets(self):
socks = []
for sock in self._sockets:
if self._is_socket_errored(sock):
socks.append(sock)
for sock in socks:
self._selector.unregister(sock)
self._sockets.remove(sock)
_RFC6555ConnectionManager._close_socket(sock)
@staticmethod
def _close_socket(sock):
try:
sock.close()
except _SOCKET_ERRORS:
pass
def _is_acceptable_errno(self, errno):
if errno == 0 or errno in _ASYNC_ERRNOS:
return True
self._error = socket.error()
self._error.errno = errno
return False
def _is_socket_errored(self, sock):
errno = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
return not self._is_acceptable_errno(errno)
def _remove_all_sockets(self):
for sock in self._sockets:
self._selector.unregister(sock)
_RFC6555ConnectionManager._close_socket(sock)
self._sockets = []