async def quote_many(num_quotes=1, conn_limit=20, progress=None, step=10): if progress is None: progress = ProgressBar() progress.max = num_quotes // step logger.info('Process total %d quotes with max %d concurrent connections' % (num_quotes, conn_limit)) logger.debug('... progress bar increment step size: %d coroutines' % step) semaphore = asyncio.Semaphore(conn_limit) coro_to_fut = asyncio.ensure_future futures = [ coro_to_fut(quote_with_lock(semaphore)) for i in range(num_quotes) ] t_start = datetime.today() for ith, fut in enumerate(asyncio.as_completed(futures), 1): if ith % step == 0: progress.next() await fut t_end = datetime.today() progress.finish() logger.info('All coroutines complete in {:.2f} seconds'.format( (t_end - t_start).total_seconds() )) quotes = [fut.result() for fut in futures] return quotes
async def save_profiles(names): conn = aiohttp.TCPConnector(limit=50, verify_ssl=False) with aiohttp.ClientSession(connector=conn) as session: ps = [Profile(name, session) for name in names] futures = [asyncio.ensure_future(p.get_info()) for p in ps] futures += [asyncio.ensure_future(p.get_publications()) for p in ps] progress, step = ProgressBar(), 10 progress.max = len(futures) // step for i, future in enumerate(asyncio.as_completed(futures), 1): if i % step == 0: progress.next() await future progress.finish() return [future.result() for future in futures]
def quote_many(num_quotes=1, conn_limit=20, progress=None, step=10): if progress is None: progress = ProgressBar() progress.max = num_quotes // step logger.info('Process total %d quotes with max %d concurrent connections' % (num_quotes, conn_limit)) logger.debug('... progress bar increment step size: %d coroutines' % step) semaphore = asyncio.Semaphore(conn_limit) # wrap coroutines with future # For Python 3.4.4+, asyncio.ensure_future(...) # will wrap coro as Task and keep input the same # if it is already Future. try: coro_to_fut = asyncio.ensure_future except AttributeError: logger.warning('asyncio.ensure_future requires Python 3.4.4+. ' 'Fall back to asyncio.async') coro_to_fut = asyncio.async futures = [ coro_to_fut(quote_with_lock(semaphore)) for i in range(num_quotes) ] t_start = datetime.today() for ith, fut in enumerate(asyncio.as_completed(futures), 1): if ith % step == 0: progress.next() yield from fut t_end = datetime.today() progress.finish() logger.info('All coroutines complete in {:.2f} seconds'.format( (t_end - t_start).total_seconds() )) quotes = [fut.result() for fut in futures] return quotes