def run_sync():
log("\nRunning synchronously with busy-waiting\n")
sock = create_socket()
request = "GET xkcd.com HTTP/1.0\r\nHost: xkcd.com\r\n\r\n"
encoded = request.encode("ascii")
wait_for_socket_in_a_loop(encoded, sock)
log("Ready!")
def wait_for_socket_in_a_loop(encoded, sock):
while True:
try:
sock.send(encoded)
break
except OSError as e:
log(f"Error: {e}")
def __init__(self, coroutine):
log("Task :: init started")
self.coroutine = coroutine
f = Future()
f.set_result(None)
self.step(f)
log("Task :: init finished")
def connected(self, key, mask):
log(f"Connected! ({self.url})")
selector.unregister(key.fd)
request = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url)
self.sock.send(request.encode('ascii'))
# Register the next callback.
selector.register(key.fd, selectors.EVENT_READ, self.read_response)
def fetch(self):
log(f"Fetcher :: Fetch started ({self.url})")
sock = yield from self.create_socket()
request = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url)
sock.send(request.encode('ascii'))
self.response = yield from self.read_all(sock)
global stopped
stopped = True
log(f"Fetcher :: Fetch finished ({self.url})")
def step(self, future):
log("Task :: step started")
try:
log("Task :: Coroutine - before send")
next_future = self.coroutine.send(future.result)
log("Task :: Coroutine - after send")
except StopIteration:
log("Task :: finished")
return
next_future.add_done_callback(self.step)
log("Task :: step finished")
def fetch(self):
log(f"Creating socket ({self.url})")
self.sock = socket.socket()
self.sock.setblocking(False)
try:
self.sock.connect(('xkcd.com', 80))
except BlockingIOError:
pass
# Register next callback.
selector.register(self.sock.fileno(), selectors.EVENT_WRITE,
self.connected)
def read(self, sock):
log(f"Fetcher :: read :: Started ({self.url})")
f = Future()
def on_readable():
log(f"Fetcher :: read :: Socket received data ({self.url})")
f.set_result(sock.recv(4096))
selector.register(sock.fileno(), selectors.EVENT_READ, on_readable)
log(f"Fetcher :: read :: Before sleeping ({self.url})")
chunk = yield from f # Read one chunk.
log(f"Fetcher :: read :: After sleeping ({self.url})")
log(f"Fetcher :: read :: Unregistering from selector ({self.url})")
selector.unregister(sock.fileno())
log(f"Fetcher :: read :: Finished ({self.url})")
return chunk
def loop(selector):
log("Starting event loop")
while True:
log("Waiting for events...")
events = selector.select()
log("Got event!")
for event_key, event_mask in events:
callback = event_key.data
callback()
break
log("Exiting event loop")
def create_socket():
sock = socket.socket()
sock.setblocking(False)
try:
log("Connect")
sock.connect(("xkcd.com", 80))
log("After connect")
except BlockingIOError as e:
log(f"Caught: {e}")
log("After try")
return sock
def read_response(self, key, mask):
log(f"Read response ({self.url})")
global stopped
chunk = self.sock.recv(4096) # 4k chunk size.
if chunk:
log(f"Got chunk ({self.url})")
self.response += chunk
else:
log(f"Done reading ({self.url})")
selector.unregister(key.fd) # Done reading.
links = self.parse_links()
for link in links.difference(seen_urls):
urls_todo.add(link)
log(f"Adding link {self.url} -> {link}")
Fetcher(link).fetch() # <- New Fetcher.
seen_urls.update(links)
urls_todo.remove(self.url)
if not urls_todo:
log(f"No more links to process ({self.url})")
stopped = True
def loop():
log("Loop :: started")
while not stopped:
events = selector.select()
log("Loop :: selected")
for event_key, event_mask in events:
callback = event_key.data
callback()
log("Loop :: finished")
def create_socket(self):
log(f"Fetcher :: Creating socket ({self.url})")
sock = socket.socket()
sock.setblocking(False)
try:
sock.connect(('xkcd.com', 80))
except BlockingIOError:
pass
f = Future()
def on_connected():
log(f"Fetcher :: Socket connected ({self.url})")
f.set_result(None)
selector.register(sock.fileno(), selectors.EVENT_WRITE, on_connected)
log(f"Fetcher :: Before sleeping ({self.url})")
yield from f
log(f"Fetcher :: After sleeping ({self.url})")
log(f"Fetcher :: Unregistering from selector ({self.url})")
selector.unregister(sock.fileno())
return sock
def read_all(self, sock):
log(f"Fetcher :: read_all :: Started ({self.url})")
response = []
log(f"Fetcher :: read_all :: Before sleeping ({self.url})")
chunk = yield from self.read(sock)
log(f"Fetcher :: read_all :: After sleeping ({self.url})")
while chunk:
log(f"Fetcher :: read_all :: Got chunk ({self.url})")
response.append(chunk)
log(f"Fetcher :: read_all :: Before sleeping ({self.url})")
chunk = yield from self.read(sock)
log(f"Fetcher :: read_all :: After sleeping ({self.url})")
log(f"Fetcher :: read_all :: Returning response ({self.url} - {response})"
)
log(f"Fetcher :: read_all :: Finished ({self.url})")
return b''.join(response)
def on_connected():
log(f"Fetcher :: Socket connected ({self.url})")
f.set_result(None)
def connected():
selector.unregister(sock.fileno())
log("Connected!")
def add_done_callback(self, fn):
self._callbacks.append(fn)
log(f"Future :: add_done_callback - {self._callbacks}")
def on_readable():
log(f"Fetcher :: read :: Socket received data ({self.url})")
f.set_result(sock.recv(4096))
def fetch(self):
log(f"Fetcher :: Fetch started ({self.url})")
log(f"Fetcher :: Creating socket ({self.url})")
sock = socket.socket()
sock.setblocking(False)
try:
sock.connect(('xkcd.com', 80))
except BlockingIOError:
pass
f = Future()
def on_connected():
log(f"Fetcher :: Socket connected ({self.url})")
f.set_result(None)
selector.register(sock.fileno(),
selectors.EVENT_WRITE,
on_connected)
log(f"Fetcher :: Before sleeping ({self.url})")
yield f
log(f"Fetcher :: After sleeping ({self.url})")
log(f"Fetcher :: Unregistering from selector ({self.url})")
selector.unregister(sock.fileno())
request = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url)
sock.send(request.encode('ascii'))
while True:
f = Future()
def on_readable():
log(f"Fetcher :: Socket received data ({self.url})")
f.set_result(sock.recv(4096))
selector.register(sock.fileno(),
selectors.EVENT_READ,
on_readable)
log(f"Fetcher :: Before sleeping ({self.url})")
chunk = yield f
log(f"Fetcher :: After sleeping ({self.url})")
log(f"Fetcher :: Unregistering from selector ({self.url})")
selector.unregister(sock.fileno())
if chunk:
log(f"Fetcher :: Got chunk ({self.url})")
self.response += chunk
else:
log(f"Fetcher :: Done reading ({self.url})")
break
global stopped
stopped = True
log(f"Fetcher :: Fetch finished ({self.url})")
def set_result(self, result):
log(f"Future :: set_result - {result}, callbacks: {self._callbacks}")
self.result = result
for fn in self._callbacks:
fn(self)
def run_async():
log("\nRunning asynchronously with a simple event loop\n")
sock = create_socket()
wait_for_socket_async(sock)
log("Ready!")
