async def quote_many(num_quotes=1, conn_limit=20, progress=None, step=10):
if progress is None:
progress = ProgressBar()
progress.max = num_quotes // step
logger.info('Process total %d quotes with max %d concurrent connections'
% (num_quotes, conn_limit))
logger.debug('... progress bar increment step size: %d coroutines' % step)
semaphore = asyncio.Semaphore(conn_limit)
coro_to_fut = asyncio.ensure_future
futures = [
coro_to_fut(quote_with_lock(semaphore))
for i in range(num_quotes)
]
t_start = datetime.today()
for ith, fut in enumerate(asyncio.as_completed(futures), 1):
if ith % step == 0:
progress.next()
await fut
t_end = datetime.today()
progress.finish()
logger.info('All coroutines complete in {:.2f} seconds'.format(
(t_end - t_start).total_seconds()
))
quotes = [fut.result() for fut in futures]
return quotes
async def save_profiles(names):
conn = aiohttp.TCPConnector(limit=50, verify_ssl=False)
with aiohttp.ClientSession(connector=conn) as session:
ps = [Profile(name, session) for name in names]
futures = [asyncio.ensure_future(p.get_info()) for p in ps]
futures += [asyncio.ensure_future(p.get_publications()) for p in ps]
progress, step = ProgressBar(), 10
progress.max = len(futures) // step
for i, future in enumerate(asyncio.as_completed(futures), 1):
if i % step == 0:
progress.next()
await future
progress.finish()
return [future.result() for future in futures]
def quote_many(num_quotes=1, conn_limit=20, progress=None, step=10):
if progress is None:
progress = ProgressBar()
progress.max = num_quotes // step
logger.info('Process total %d quotes with max %d concurrent connections'
% (num_quotes, conn_limit))
logger.debug('... progress bar increment step size: %d coroutines' % step)
semaphore = asyncio.Semaphore(conn_limit)
# wrap coroutines with future
# For Python 3.4.4+, asyncio.ensure_future(...)
# will wrap coro as Task and keep input the same
# if it is already Future.
try:
coro_to_fut = asyncio.ensure_future
except AttributeError:
logger.warning('asyncio.ensure_future requires Python 3.4.4+. '
'Fall back to asyncio.async')
coro_to_fut = asyncio.async
futures = [
coro_to_fut(quote_with_lock(semaphore))
for i in range(num_quotes)
]
t_start = datetime.today()
for ith, fut in enumerate(asyncio.as_completed(futures), 1):
if ith % step == 0:
progress.next()
yield from fut
t_end = datetime.today()
progress.finish()
logger.info('All coroutines complete in {:.2f} seconds'.format(
(t_end - t_start).total_seconds()
))
quotes = [fut.result() for fut in futures]
return quotes
