def run_sync(): log("\nRunning synchronously with busy-waiting\n") sock = create_socket() request = "GET xkcd.com HTTP/1.0\r\nHost: xkcd.com\r\n\r\n" encoded = request.encode("ascii") wait_for_socket_in_a_loop(encoded, sock) log("Ready!")
def wait_for_socket_in_a_loop(encoded, sock): while True: try: sock.send(encoded) break except OSError as e: log(f"Error: {e}")
def __init__(self, coroutine): log("Task :: init started") self.coroutine = coroutine f = Future() f.set_result(None) self.step(f) log("Task :: init finished")
def connected(self, key, mask): log(f"Connected! ({self.url})") selector.unregister(key.fd) request = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url) self.sock.send(request.encode('ascii')) # Register the next callback. selector.register(key.fd, selectors.EVENT_READ, self.read_response)
def fetch(self): log(f"Fetcher :: Fetch started ({self.url})") sock = yield from self.create_socket() request = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url) sock.send(request.encode('ascii')) self.response = yield from self.read_all(sock) global stopped stopped = True log(f"Fetcher :: Fetch finished ({self.url})")
def step(self, future): log("Task :: step started") try: log("Task :: Coroutine - before send") next_future = self.coroutine.send(future.result) log("Task :: Coroutine - after send") except StopIteration: log("Task :: finished") return next_future.add_done_callback(self.step) log("Task :: step finished")
def fetch(self): log(f"Creating socket ({self.url})") self.sock = socket.socket() self.sock.setblocking(False) try: self.sock.connect(('xkcd.com', 80)) except BlockingIOError: pass # Register next callback. selector.register(self.sock.fileno(), selectors.EVENT_WRITE, self.connected)
def read(self, sock): log(f"Fetcher :: read :: Started ({self.url})") f = Future() def on_readable(): log(f"Fetcher :: read :: Socket received data ({self.url})") f.set_result(sock.recv(4096)) selector.register(sock.fileno(), selectors.EVENT_READ, on_readable) log(f"Fetcher :: read :: Before sleeping ({self.url})") chunk = yield from f # Read one chunk. log(f"Fetcher :: read :: After sleeping ({self.url})") log(f"Fetcher :: read :: Unregistering from selector ({self.url})") selector.unregister(sock.fileno()) log(f"Fetcher :: read :: Finished ({self.url})") return chunk
def loop(selector): log("Starting event loop") while True: log("Waiting for events...") events = selector.select() log("Got event!") for event_key, event_mask in events: callback = event_key.data callback() break log("Exiting event loop")
def create_socket(): sock = socket.socket() sock.setblocking(False) try: log("Connect") sock.connect(("xkcd.com", 80)) log("After connect") except BlockingIOError as e: log(f"Caught: {e}") log("After try") return sock
def read_response(self, key, mask): log(f"Read response ({self.url})") global stopped chunk = self.sock.recv(4096) # 4k chunk size. if chunk: log(f"Got chunk ({self.url})") self.response += chunk else: log(f"Done reading ({self.url})") selector.unregister(key.fd) # Done reading. links = self.parse_links() for link in links.difference(seen_urls): urls_todo.add(link) log(f"Adding link {self.url} -> {link}") Fetcher(link).fetch() # <- New Fetcher. seen_urls.update(links) urls_todo.remove(self.url) if not urls_todo: log(f"No more links to process ({self.url})") stopped = True
def loop(): log("Loop :: started") while not stopped: events = selector.select() log("Loop :: selected") for event_key, event_mask in events: callback = event_key.data callback() log("Loop :: finished")
def create_socket(self): log(f"Fetcher :: Creating socket ({self.url})") sock = socket.socket() sock.setblocking(False) try: sock.connect(('xkcd.com', 80)) except BlockingIOError: pass f = Future() def on_connected(): log(f"Fetcher :: Socket connected ({self.url})") f.set_result(None) selector.register(sock.fileno(), selectors.EVENT_WRITE, on_connected) log(f"Fetcher :: Before sleeping ({self.url})") yield from f log(f"Fetcher :: After sleeping ({self.url})") log(f"Fetcher :: Unregistering from selector ({self.url})") selector.unregister(sock.fileno()) return sock
def read_all(self, sock): log(f"Fetcher :: read_all :: Started ({self.url})") response = [] log(f"Fetcher :: read_all :: Before sleeping ({self.url})") chunk = yield from self.read(sock) log(f"Fetcher :: read_all :: After sleeping ({self.url})") while chunk: log(f"Fetcher :: read_all :: Got chunk ({self.url})") response.append(chunk) log(f"Fetcher :: read_all :: Before sleeping ({self.url})") chunk = yield from self.read(sock) log(f"Fetcher :: read_all :: After sleeping ({self.url})") log(f"Fetcher :: read_all :: Returning response ({self.url} - {response})" ) log(f"Fetcher :: read_all :: Finished ({self.url})") return b''.join(response)
def on_connected(): log(f"Fetcher :: Socket connected ({self.url})") f.set_result(None)
def connected(): selector.unregister(sock.fileno()) log("Connected!")
def add_done_callback(self, fn): self._callbacks.append(fn) log(f"Future :: add_done_callback - {self._callbacks}")
def on_readable(): log(f"Fetcher :: read :: Socket received data ({self.url})") f.set_result(sock.recv(4096))
def fetch(self): log(f"Fetcher :: Fetch started ({self.url})") log(f"Fetcher :: Creating socket ({self.url})") sock = socket.socket() sock.setblocking(False) try: sock.connect(('xkcd.com', 80)) except BlockingIOError: pass f = Future() def on_connected(): log(f"Fetcher :: Socket connected ({self.url})") f.set_result(None) selector.register(sock.fileno(), selectors.EVENT_WRITE, on_connected) log(f"Fetcher :: Before sleeping ({self.url})") yield f log(f"Fetcher :: After sleeping ({self.url})") log(f"Fetcher :: Unregistering from selector ({self.url})") selector.unregister(sock.fileno()) request = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url) sock.send(request.encode('ascii')) while True: f = Future() def on_readable(): log(f"Fetcher :: Socket received data ({self.url})") f.set_result(sock.recv(4096)) selector.register(sock.fileno(), selectors.EVENT_READ, on_readable) log(f"Fetcher :: Before sleeping ({self.url})") chunk = yield f log(f"Fetcher :: After sleeping ({self.url})") log(f"Fetcher :: Unregistering from selector ({self.url})") selector.unregister(sock.fileno()) if chunk: log(f"Fetcher :: Got chunk ({self.url})") self.response += chunk else: log(f"Fetcher :: Done reading ({self.url})") break global stopped stopped = True log(f"Fetcher :: Fetch finished ({self.url})")
def set_result(self, result): log(f"Future :: set_result - {result}, callbacks: {self._callbacks}") self.result = result for fn in self._callbacks: fn(self)
def run_async(): log("\nRunning asynchronously with a simple event loop\n") sock = create_socket() wait_for_socket_async(sock) log("Ready!")