async def download_one(session, cc, base_url, semaphore, verbose):
"""委托生成器函数, 驱动 get_flag 协程函数
"""
try:
# semaphore 是一个 asyncio.Semaphore 的实例, 用来控制最大并发数
# 因为它是 asyncio 的对象, 需要在前面 加 async
# 等价于 with (yield from semaphore)
async with semaphore:
# 等价于 image = yield from get_flag(...)
image = await get_flag(session, base_url, cc)
# 退出上面的 with 语句时信号量减1, 使得某一个挂起的生成器函数重新运行
# 下面的部分处理了 get_flag 抛出的异常:
# 1.对于 aiohttp.web.HTTPNotFound, 吞掉它.
# 2.其他异常用 FetchError 异常类封装后向上抛出
except web.HTTPNotFound:
status = HTTPStatus.not_found
msg = 'not found'
except Exception as exc:
# 注意这里的 raise X from Y 语法
raise FetchError(cc) from exc
else:
# !! 注意到 文件写入操作也是IO操作. 目前没有异步非阻塞写入文件的库函数... 因此他会阻塞事件循环
# 解决方案: asyncio.run_in_executor(...), 这是事件循环中的线程池
"""
loop = asyncio.get_event_loop()
# run_in_executor(...) 的第一个参数接受一个executor的实例, 如果为None则会用默认的线程池
loop.run_in_executor(None, save_flag, image, cc.lower() + '.gif')
"""
save_flag(image, cc.lower() + '.gif')
status = HTTPStatus.ok
msg = 'OK'
if verbose and msg:
print(cc, msg)
return Result(status, cc)
def download_one(cc, base_url, verbose=False):
try:
image = get_flag(base_url, cc)
except requests.exceptions.HTTPError as exc: #2 捕捉HTTPError异常, 并且对404做特殊处理(status_code是404的异常不会被抛出)
res = exc.response
if res.status_code == 404:
status = HTTPStatus.not_found #3
msg = 'not found'
else: #4 其他异常向外抛出
raise
else:
save_flag(image, cc.lower() + '.gif')
status = HTTPStatus.ok
msg = 'OK'
if verbose:
print(cc, msg)
return Result(status, cc) # 注意返回的数据结构, 值得借鉴!
def download_one(cc, base_url, semaphore, verbose):
try:
with (yield from semaphore):
image = yield from get_flag(base_url, cc)
except web.HTTPNotFound:
status = HTTPStatus.not_found
msg = 'not found'
except Exception as exc:
raise FetchError(cc) from exc
else:
save_flag(image, cc.lower() + '.gif') # 블로킹 함수
status = HTTPStatus.ok
msg = 'OK'
if verbose and msg:
print(cc, msg)
return Result(status, cc)
async def download_one(session, cc, base_url, semaphore, verbose):
try:
async with semaphore:
image = await get_flag(session, base_url, cc)
except web.HTTPNotFound:
status = HTTPStatus.not_found
msg = 'not found'
except Exception as exc:
raise FetchError(cc) from exc
else:
save_flag(image, cc.lower() + '.gif')
status = HTTPStatus.ok
msg = 'ok'
if verbose and msg:
print(cc, msg)
return Result(status, cc)
def download_one(cc, base_url, verbose=False):
try:
image = get_flag(base_url, cc)
except requests.exceptions.HTTPError as exc:
res = exc.response
if res.status_code == 404:
stats = HTTPStatus.not_found
msg = 'not found'
else:
raise
else:
save_flag(image, cc.lower()+'.gif')
status = HTTPStatus.ok
msg = 'OK'
if verbose:
print(cc, msg)
return Result(status, cc)