async def do_loop(urls): async with aiohttp.ClientSession(loop=loop) as session: async def fetch(url, i=0): with aiohttp.Timeout(10, loop=session.loop): logger.info("fetch(%s): url=%s", i, url) async with session.get(url) as response: text = await response.text() urls = [spiderlib.urljoin(url, spiderlib.remove_fragment(new_url)) for new_url in spiderlib.get_links(text)] logger.info("fetched(%s): url=%s", i, url) return urls semaphore = asyncio.Semaphore(10) async def access(url): async with semaphore: return await cacher(lambda url: dispatcher.dispatch(fetch, url), url) cacher = lib.Cacher(lambda url: url) dispatcher = lib.LimitterDispatcher(lambda url: spiderlib.urlparse(url).netloc, limit_count=2) q = asyncio.Queue() rq = lib.RecQueue(q) for url in urls: rq.add(access(url)) await rq.join() while not q.empty(): urls = await q.get() for url in urls: rq.add(access(url)) await rq.join() print("yay", rq.ec)
def limited_fetch(): cacher = lib.Cacher(lambda request: (request.domain, request.args)) dispatcher = lib.LimitterDispatcher(lambda request: request.domain, limit_count=3) def coro_fn(request): return dispatcher.dispatch(lib.mock_fetch, request) # return cacher(lambda request: dispatcher.dispatch(lib.mock_fetch, request), request) return coro_fn
async def do_loop(requests): dispatcher = lib.LimitterDispatcher(lambda request: request.domain, limit_count=2) rq = _RecQueue( lambda response: [dispatcher.dispatch(lib.mock_fetch, r) for r in response.get_links()]) for request in requests: rq.add(dispatcher.dispatch(lib.mock_fetch, request)) await rq.join()
async def do_loop(requests): dispatcher = lib.LimitterDispatcher(lambda request: request.domain, limit_count=2) todo = [] done = [] st = time.time() for r in requests: response = await dispatcher.dispatch(lib.mock_fetch, r) done.append(r) todo.extend(response.get_links()) while todo: r = todo.pop() response = await dispatcher.dispatch(lib.mock_fetch, r) done.append(r) todo.extend(response.get_links()) logger.info("takes %s, total %s", time.time() - st, len(done))
async def do_loop(requests): dispatcher = lib.LimitterDispatcher(lambda request: request.domain, limit_count=2) semaphore = asyncio.Semaphore(10) q = asyncio.Queue() async def fetch_urls(request): async with semaphore: response = await dispatcher.dispatch(lib.mock_fetch, request) return response.get_links() async def do_work(): while True: item = await q.get() links = await fetch_urls(item) q.task_done() for link in links: await q.put(link) workers = [] workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) workers.append(asyncio.ensure_future(do_work())) st = time.time() for req in requests: await q.put(req) while not (q.empty() and q._unfinished_tasks <= 0): # xxx logger.info("loop: unfinished=%s, queue=%s", q._unfinished_tasks, len(q._queue)) await asyncio.sleep(0.2) for w in workers: w.cancel() logger.info("takes %s, total %s", time.time() - st, "x")
async def do_loop(requests): dispatcher = lib.LimitterDispatcher(lambda request: request.domain, limit_count=2) semaphore = asyncio.Semaphore(10) todo = asyncio.Queue() done = asyncio.Queue() async def to_future(request): async with semaphore: return await dispatcher.dispatch(lib.mock_fetch, request) async def done_worker(): while True: request = await done.get() fut = asyncio.ensure_future(to_future(request)) todo.put_nowait(fut) async def todo_worker(): def callback(fut): todo.task_done() response = fut.result() for request in response.get_links(): done.put_nowait(request) while True: fut = await todo.get() fut.add_done_callback(callback) workers = [] workers.append(asyncio.ensure_future(done_worker())) workers.append(asyncio.ensure_future(todo_worker())) st = time.time() for req in requests: done.put_nowait(req) while not (todo._unfinished_tasks == 0 and len(done._queue) == 0): print("loop") logger.info("loop: todo=%s, done=%s", todo._unfinished_tasks, len(done._queue)) await asyncio.sleep(0.2) for w in workers: w.cancel() logger.info("takes %s, total %s", time.time() - st, done._unfinished_tasks)
async def do_loop(loop): cacher = lib.Cacher(lambda url: url) dispatcher = lib.LimitterDispatcher( lambda url: spiderlib.urlparse(url).netloc, limit_count=2) async with aiohttp.ClientSession(loop=loop) as session: async def fetch(url): with aiohttp.Timeout(10, loop=session.loop): async with session.get(url) as response: text = await response.text() urls = [ spiderlib.urljoin(url, spiderlib.remove_fragment(new_url)) for new_url in spiderlib.get_links(text) ] return url, urls async def consume(supervisor, pipe, i): url = await supervisor.pipe.read() if url is lib.END_OF_STREAM: return logger.info("fetch(%s): url=%s", i, url) response = await supervisor.consume( cacher, lambda request: dispatcher.dispatch(fetch, request), url) pipe.write_nowait(response) async def provide(supervisor, pipe, i): st = time.time() url, urls = await pipe.read() logger.info("fetched(%s): url=%s", i, url) for u in urls: pipe.write_nowait(u) logger.info("takes (%s): %s", i, time.time() - st) pipe = lib.Pipe(asyncio.Queue(), asyncio.Queue()) supervisor = lib.Supervisor(pipe, provide, consume, concurrency=10) base_url = 'http://www.tornadoweb.org/en/stable/' base_url2 = 'http://python.org/' await supervisor.run_loop([base_url, base_url2])
async def do_loop(requests): dispatcher = lib.LimitterDispatcher(lambda request: request.domain, limit_count=2) cacher = lib.Cacher(lambda request: (request.domain, request.args)) rq = lib.RecQueue(asyncio.Queue()) for request in requests: rq.add( cacher( lambda request: dispatcher.dispatch(lib.mock_fetch, request), request)) await rq.join() while not rq.empty(): response = await rq.get() for request in response.get_links(): rq.add( cacher( lambda request: dispatcher.dispatch( lib.mock_fetch, request), request)) await rq.join() print(vars(rq.q)) print("yay", rq.ec)