def the_asynchronous_process():
import asyncio
async def worker_1():
print('worker_1 start')
await asyncio.sleep(1)
print('worker_1 done')
async def worker_2():
print('worker_2 start')
await asyncio.sleep(2)
print('worker_2 done')
async def main():
task1 = asyncio.ensure_future(worker_1())
task2 = asyncio.ensure_future(worker_2())
print('before await')
await task1
print('awaited worker_1')
await task2
print('awaited worker_2')
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
try:
task = loop.create_task(main())
finally:
loop.close()
loop.run_until_complete(task)
print('TIME: ', now() - start)
def the_synchronous_process():
import asyncio
async def worker_1():
print('worker_1 start')
await asyncio.sleep(1)
print('worker_1 done')
async def worker_2():
print('worker_2 start')
await asyncio.sleep(2)
print('worker_2 done')
async def main():
print('before await')
await worker_1()
print('awaited worker_1')
await worker_2()
print('awaited worker_2')
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
print('TIME: ', now() - start)
def get_movies_from_douban_with_asynchronous():
import asyncio
import aiohttp
from bs4 import BeautifulSoup
header = header = {
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; Win64; x64)"
" AppleWebKit/537.36 (KHTML, like Gecko)"
" Chrome/74.0.3729.157 Safari/537.36"
}
async def fetch_content(url):
async with aiohttp.ClientSession(
headers=header,
connector=aiohttp.TCPConnector(ssl=False)) as session:
async with session.get(url) as response:
return await response.text()
async def main():
url = "https://movie.douban.com/cinema/later/beijing/"
init_page = await fetch_content(url)
init_soup = BeautifulSoup(init_page, 'lxml')
movie_names, urls_to_fetch, movie_dates = [], [], []
all_movies = init_soup.find('div', id="showing-soon")
for each_movie in all_movies.find_all('div', class_="item"):
all_a_tag = each_movie.find_all('a')
all_li_tag = each_movie.find_all('li')
movie_names.append(all_a_tag[1].text)
urls_to_fetch.append(all_a_tag[1]['href'])
movie_dates.append(all_li_tag[0].text)
tasks = [fetch_content(url) for url in urls_to_fetch]
pages = await asyncio.gather(*tasks)
for movie_name, movie_date, page in zip(movie_names, movie_dates,
pages):
soup_item = BeautifulSoup(page, 'lxml')
img_tag = soup_item.find('img')
print('{} {} {}'.format(movie_name, movie_date,
img_tag['src']))
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
task = loop.create_task(main())
try:
loop.run_until_complete(task)
finally:
loop.close()
print('TIME: ', now() - start)
def customer_and_produce_model():
import asyncio
import random
async def consumer(queue, id):
while True:
val = await queue.get()
print('{} get a val: {}'.format(id, val))
await asyncio.sleep(1)
async def producer(queue, id):
for i in range(5):
val = random.randint(1, 10)
await queue.put(val)
print('{} put a val: {}'.format(id, val))
await asyncio.sleep(1)
async def main():
queue = asyncio.Queue()
consumer_1 = asyncio.ensure_future(consumer(queue, 'consumer_1'))
consumer_2 = asyncio.ensure_future(consumer(queue, 'consumer_2'))
producer_1 = asyncio.ensure_future(producer(queue, 'producer_1'))
producer_2 = asyncio.ensure_future(producer(queue, 'producer_2'))
await asyncio.sleep(10)
consumer_1.cancel()
consumer_2.cancel()
await asyncio.gather(consumer_1,
consumer_2,
producer_1,
producer_2,
return_exceptions=True)
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
task = loop.create_task(main())
loop.run_until_complete(task)
print('TIME: ', now() - start)
def synchronous_execution_python36():
import asyncio
async def crawl_page(url):
print('crawling {}'.format(url))
sleep_time = int(url.split('_')[-1])
await asyncio.sleep(sleep_time)
print('OK {}'.format(url))
async def main(urls):
for url in urls:
await crawl_page(url)
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(['url_1', 'url_2', 'url_3', 'url_4']))
finally:
loop.close()
print('TIME: ', now() - start)
def limit_runtime_and_set_error_handling_for_coroutine_tasks():
import asyncio
async def worker_1():
await asyncio.sleep(1)
return 1
async def worker_2():
await asyncio.sleep(2)
return 2 / 0
async def worker_3():
await asyncio.sleep(3)
return 3
async def main():
task_1 = asyncio.ensure_future(worker_1())
task_2 = asyncio.ensure_future(worker_2())
task_3 = asyncio.ensure_future(worker_3())
# 超过 2s 则取消任务三
await asyncio.sleep(2)
task_3.cancel()
# 不设置 return_exceptions = True，错误会被抛出，需要使用 try except
# 捕获，进而造成其他还没被执行的任务会被完全取消掉
res = await asyncio.gather(task_1,
task_2,
task_3,
return_exceptions=True)
print(res)
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
task = loop.create_task(main())
loop.run_until_complete(task)
print('TIME: ', now() - start)
def add_callback_function():
import asyncio
async def worker_1():
print('worker_1 start')
await asyncio.sleep(1)
print('worker_1 done')
async def worker_2():
print('worker_2 start')
await asyncio.sleep(2)
print('worker_2 done')
async def main():
task1 = asyncio.ensure_future(worker_1())
task2 = asyncio.ensure_future(worker_2())
task1.add_done_callback(callback_function)
task2.add_done_callback(callback_function)
print('before await')
await task1
print('awaited worker_1')
await task2
print('awaited worker_2')
def callback_function(future):
print('invoked callback function')
print('Callback: ', future.result())
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
task = loop.create_task(main())
try:
loop.run_until_complete(task)
finally:
loop.close()
print('TIME: ', now() - start)
def asynchronous_execution_python36():
import asyncio
async def crawl_page(url):
print('crawling {}'.format(url))
sleep_time = int(url.split('_')[-1])
await asyncio.sleep(sleep_time)
print('OK {}'.format(url))
async def main(urls):
tasks = [asyncio.ensure_future(crawl_page(url)) for url in urls]
for task in tasks:
await task
# await asyncio.gather(*tasks)
from win32.timezone import now
start = now()
loop = asyncio.get_event_loop()
task = loop.create_task(main(['url_1', 'url_2', 'url_3', 'url_4']))
try:
loop.run_until_complete(task)
finally:
loop.close()
print('TIME: ', now() - start)