def download_many(cc_list):
# ワーカースレッド数を設定します。最大スレッド数（MAX_WORKERS）と
# 実際の処理の対象となる要素の数のどちらか小さい方の値を使用すれば、不要なスレッドは作成されません。
workers = min(MAX_WORKERS, len(cc_list))
# ワーカースレッド数を指定してThreadPoolExecutorをインスタンス化します。
# executor.__exit__メソッドから呼び出されるexecutor.shutdown(wait=True)は、
# すべてのスレッドが完了するまでブロックされます。
with futures.ThreadPoolExecutor(workers) as executor:
# mapメソッドは組み込み関数のmapと似ていますが、
# 指定のdownload_one関数が複数のスレッドから並行して呼び出されるところが異なります。
# このメソッドは、それぞれの関数が返した値を反復処理で取得できるジェネレータを返します。
res = executor.map(download_one, sorted(cc_list))
# 得られた結果の数を返します。スレッド化された呼び出しが例外を上げると、
# イテレータから対応する戻り値を取得しようとした暗黙的なnext()呼び出しの例外として、ここで上げられます。
return len(list(res))
if __name__ == '__main__':
# download_manyの改訂版を指定して、flagsモジュールのmain関数を呼び出します。
main(download_many)
future = executor.submit(download_one, cc)
to_do.append(future)
msg = 'Scheduled for {}: {}'
print(msg.format(cc, future))
results = []
for future in futures.as_completed(to_do):
res = future.result()
msg = '{} result: {!r}'
print(msg.format(future, res))
results.append(res)
return len(results)
#IO密集型操作
# def download_many(cc_list):
# workers = min(MAX_WORKERS, len(cc_list))
# with futures.ThreadPoolExecutor(workers) as executor:
# res = executor.map(download_one, sorted(cc_list))
# return len(list(res))
#多进程方式
# def download_many(cc_list):
# with futures.ProcessPoolExecutor(workers) as executor:
# res = executor.map(download_one, sorted(cc_list))
# return len(list(res))
if __name__ == '__main__':
main(download_many)
@asyncio.coroutine
def download_one(cc):
image = yield from get_flag(cc)
print('{} retrieved.'.format(cc))
save_flag(image, cc.lower() + '.gif')
return cc
@asyncio.coroutine
def downloader_coro(cc_list):
to_do = [download_one(cc) for cc in cc_list]
results = []
for future in asyncio.as_completed(to_do):
print(future)
result = yield from future
results.append(result)
return results
def download_many(cc_list):
loop = asyncio.get_event_loop()
results = loop.run_until_complete(downloader_coro(cc_list))
loop.close()
return len(results)
if __name__ == '__main__':
main(download_many)
US retrieved.
IR retrieved.
20 flags downloaded in 0.93s
"""
from concurrent import futures
from flags import save_flag, get_flag, show, main # <1>
MAX_WORKERS = 20 # <2>
def download_one(cc): # <3>
image = get_flag(cc)
show(cc)
save_flag(image, cc.lower() + '.gif')
return cc
def download_many(cc_list):
workers = min(MAX_WORKERS, len(cc_list)) # <4>
with futures.ThreadPoolExecutor(workers) as executor: # <5>
res = executor.map(download_one, sorted(cc_list)) # <6>
return len(list(res)) # <7>
if __name__ == '__main__':
main(download_many) # <8>
$ python3 flags_threadpool.py
BR BD EG CN ID DE IN JP ET NG VN RU CD IR MX PH PK US TR FR
20 flags downloaded in 0.93s
"""
from concurrent import futures
from flags import save_flag, get_flag, show, main # <1>
MAX_WORKERS = 20 # <2>
def download_one(cc): # <3>
image = get_flag(cc)
show(cc)
save_flag(image, cc.lower() + '.gif')
return cc
def download_many(cc_list):
workers = min(MAX_WORKERS, len(cc_list)) # <4>
with futures.ThreadPoolExecutor(workers) as executor: # <5>
res = executor.map(download_one, sorted(cc_list)) # <6>
return len(list(res)) # <7>
if __name__ == '__main__':
main(download_many) # <8>
assert resp.status == 200
return await resp.read() # <5>
async def download_one(client, cc): # <6>
image = await get_flag(client, cc) # <7>
show(cc)
save_flag(image, cc.lower() + '.gif')
return cc
async def download_many(loop, cc_list):
tcpconnector = aiohttp.TCPConnector(family=socket.AF_INET)
async with aiohttp.ClientSession(connector=tcpconnector) as client:
# async with aiohttp.ClientSession(loop=loop) as client: # <8>
to_do = [download_one(client, cc) for cc in sorted(cc_list)] # <9>
res = await asyncio.gather(*to_do)
return len(res) # <10>
def start(cc_list):
loop = asyncio.get_event_loop() # <11>
res = loop.run_until_complete(download_many(loop, cc_list)) # <12>
loop.close() # <13>
return res
if __name__ == '__main__':
main(start)
# END FLAGS_ASYNCIO
$ python3 flags_threadpool.py
DE FR BD CN EG RU IN TR VN ID JP BR NG MX PK ET PH CD US IR
20 downloads in 0.35s
"""
# tag::FLAGS_THREADPOOL[]
from concurrent import futures
from flags import save_flag, get_flag, main # <1>
def download_one(cc: str): # <2>
image = get_flag(cc)
print(cc, end=' ', flush=True)
save_flag(image, cc.lower() + '.gif')
return cc
def download_many(cc_list: list[str]) -> int:
with futures.ThreadPoolExecutor() as executor: # <3>
res = executor.map(download_one, sorted(cc_list)) # <4>
return len(list(res)) # <5>
if __name__ == '__main__':
main(download_many) # <6>
# end::FLAGS_THREADPOOL[]
#!/usr/bin/env python
__date__ = '2015-09-23'
from concurrent import futures
from flags import save_flag, get_flag, show, main
MAX_WORKERS = 20
def dl_one(cc):
image = get_flag(cc)
show(cc)
save_flag(image, cc.lower() + '.gif')
return cc
def dl_many(cc_list):
workers = min(MAX_WORKERS, len(cc_list))
with futures.ThreadPoolExecutor(workers) as executor:
res = executor.map(dl_one, sorted(cc_list))
return len(list(res))
if __name__ == '__main__':
# call the main function from the flags module(flags.py), passing the enhanced ver of dl_many
main(dl_many)
def supervisor():#调用线程，显示对象，杀死线程
signal = Signal
spinner = threading.Thread(target=spin, args=('thinking!', signal))
print('spinner object:', spinner)
spinner.start()
result = slow_function()#运行 slow_function 函数，阻塞主线程。
signal.go = False#设置信号
spinner.join()#等待线程自己退出
return result
def main():
result = supervisor()
print('Answer:', result)
if __name__ == '__main__':
main()
使用asyncio协程实现
import asyncio
import itertools
import sys
@asyncio.coroutine
def spin(msg):
write, flush = sys.stdout.write, sys.stdout.flush
for char in itertools.cycle('|/-\\')
status = char + ' ' + msg
write(status)
flush()
write('\x08' * len(status))
def download_one(
cc
): # Function to download a single image; this is what each thread will execute.
image = get_flag(cc)
show(cc)
save_flag(image, cc.lower() + '.gif')
return cc
def download_many(cc_list):
workers = min(
MAX_WORKERS, len(cc_list)
) # Set the number of worker threads; use the smaller number between the maximum we want to allow (MAX_WORKERS) and the actual items to be processed, so noo unnecessary threads are created.
with futures.ThreadPoolExecutor(
workers
) as executor: # Instantiate the ThreadPoolExecutor with that number of worker threads; the executor.__exit__ method will call executor.shutdown(wait=True), which will block until all threads are done.
res = executor.map(
download_one, sorted(cc_list)
) # The map method is similar to the map built-in, except that the download_one function will be called concurrently from multiple threads; it returns a generator that can be iterated over to retrieve the value returned by each function.
return len(
list(res)
) # Return the number of results obtained; if any of the threaded calls raised an exception, that exception would be raised here as the implicit next() call tried to retrieve the corresponding return value from the iterator.
if __name__ == '__main__':
main(
download_many
) # Call the main function from the flags module, passing the enhanced version of download_many.
def download_one(cc):
image = get_flag(cc)
show(cc)
save_flag(image, cc.lower() + '.gif')
return cc
def download_many_t(cc_list):
workers = min(MAX_WORKERS, len(cc_list))
with futures.ThreadPoolExecutor(workers) as executor:
res = executor.map(download_one, sorted(cc_list))
return len(list(res))
def download_many_p(cc_list):
with futures.ProcessPoolExecutor() as executor:
res = executor.map(download_one, sorted(cc_list))
return len(list(res))
if __name__ == '__main__':
arg = sys.argv[-1]
if arg.endswith('p'):
print('Using ProcessPool')
main(download_many_p)
else:
print('Using ThreadPool')
main(download_many_t)
with futures.ThreadPoolExecutor(workers) as executor:
res = executor.map(
download_one,
sorted(cc_list)) # 至少要等到第一个任务结束才能获取结果(按照Schedule顺序获取结果)
return len(list(res))
def download_many_ac(cc_list):
cc_list = cc_list[:5]
with futures.ThreadPoolExecutor(max_workers=3) as executor:
to_do = []
for cc in sorted(cc_list):
future = executor.submit(download_one, cc)
to_do.append(future)
msg = 'Scheduled for {}:{}'
print(msg.format(cc, future))
results = []
for future in futures.as_completed(
to_do): # yields futures when completed
res = future.result()
msg = '{} result: {!r}'
print(msg.format(future, res))
results.append(res)
return len(results)
if __name__ == '__main__':
print('futures中的线程池方式并发')
main(download_many) # download_many_ac
singleUniv = []
for td in ltd:
singleUniv.append(td.string)
allUniv.append(singleUniv)
def printUnivList(num):
print("{:^4}{:^10}{:^5}{:^8}{:^10}".format("排名","学校名称","省市","总分","培养规模"))
for i in range(num):
u=allUniv[i]
print("{:^4}{:^10}{:^5}{:^8}{:^10}".format(u[0],u[1],u[2],u[3],u[6]))
def main():
url = 'http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html'
html = getHTMLText(url)
soup = BeautifulSoup(html, "html.parser")
fillUnivList(soup)
printUnivList(10)
main()
# %%
#e21.1AutoKeywordSearch.py
import requests
from bs4 import BeautifulSoup
import re
import json
def getKeywordResult(keyword):
url = 'http://www.baidu.com/s?wd='+keyword
try:
r = requests.get(url, timeout=30)
r.raise_for_status()
r.encoding = 'utf-8'
return r.text
image = get_flag(cc)
show(cc)
# print(name,end=' ') # 只是为了显示传参数
sys.stdout.flush()
save_flag(image, cc.lower() + '.gif')
return cc
# Thread
def download_many(cc_list):
workers = min(MAX_WORKERS, len(cc_list)) # <4>
with futures.ThreadPoolExecutor(workers) as executor: # <5>
res = executor.map(download_one, sorted(cc_list),sorted(cc_list)) # <6>
# return len(list(res)) # <7>
# Process
def download_many_Process(cc_list):
workers = min(MAX_WORKERS, len(cc_list)) # <4>
with futures.ProcessPoolExecutor() as executor: # <5>
res = executor.map(download_one, sorted(cc_list),sorted(cc_list)) # <6>
# return len(list(res)) # <7>
if __name__ == '__main__':
main(download_many_Process) # <8>
# main(download_many) # <8>
# END FLAGS_THREADPOOL
import asyncio
import aiohttp
import time
from flags import BASE_URL, save_flag, show, main
async def get_flag(cc):
url = "{}/{cc}/{cc}.gif".format(BASE_URL, cc=cc.lower())
resp = await aiohttp.request("GET", url)
image = await resp.read()
return image
async def down_loadone(cc):
image = await get_flag(cc)
show(cc)
save_flag(image, cc.lower() + ".gif")
return cc
async def dowload_many(cc_list):
tasks = [down_loadone(cc) for cc in sorted(cc_list)]
asyncio.run(asyncio.wait(tasks))
return len(cc_list)
if __name__ == '__main__':
start = time.time()
main(dowload_many)
print("---->", time.time() - start)
