async def main():
# 此批访问的并发数为60, 失败request的返回空字符串
manager = AsyncRequestManager(parral_amount=60, fail_return='')
# 此批访问全部服用此connection
connection = await AsyncConnection.create()
# 构建100个请求、其中第五个故意设置为无效的url
for i in range(100):
url = valid_url
if i == 5: url = invalid_url
request = Request(
http_conn=connection,
method='GET',
url=url
)
manager.add_request(request)
# 访问全部，resp: ['each resp text'], error
resp, error = await manager.visit_all()
suc, fail = 0, 0
for i in resp:
if len(i) > 0:
suc += 1
else:
fail += 1
print(suc, fail)
assert fail == 1
connection.close()
async def main():
manager = AsyncRequestManager(parral_amount=2)
# 显式调用set_log方法，设置日志相关参数即打开了日志记录
manager.set_log(
name='http_request_log',
dir_path='./',
clear=True,
clear_days=60
)
connection = await AsyncConnection.create()
for i in range(10):
url = valid_url
if i == 5: url = invalid_url # a fail request
request = Request(
http_conn=connection,
method='GET',
url=url,
log_flag=f'req{i}',
)
manager.add_request(request)
resp, error = await manager.visit_all()
for i in resp:
print('suc' if len(i) > 0 else 'fail')
connection.close()
def main():
manager = SyncRequestManager(parral_amount=2, cancel_if_fail=True)
connection = SyncConnection.create()
for i in range(10):
url = valid_url
if i == 5: url = invalid_url
request = Request(http_conn=connection, method='GET', url=url)
manager.add_request(request)
resp, error = manager.visit_all()
print(resp, type(resp), error)
connection.close()
def main():
manager = SyncRequestManager(parral_amount=3, fail_return='')
connection = SyncConnection.create()
for i in range(10):
url = valid_url
if i == 5: url = invalid_url
request = Request(http_conn=connection, method='GET', url=url)
manager.add_request(request)
resp, error = manager.visit_all()
for i in resp:
print('-->', 'suc' if len(i) > 0 else 'fail')
connection.close()
def main():
manager = SyncRequestManager(parral_amount=4)
connection = SyncConnection.create()
for i in range(10):
url = valid_url
if i == 5: url = invalid_url
request = Request(http_conn=connection, method='GET', url=url)
manager.add_request(request)
cursor_resp = manager.visit_iter()
for resp in cursor_resp:
print('suc' if len(resp) > 0 else 'fail')
connection.close()
async def main():
manager = AsyncRequestManager(parral_amount=4)
connection = await AsyncConnection.create()
for i in range(10):
url = valid_url
if i == 5: url = invalid_url # a fail request
request = Request(http_conn=connection, method='GET', url=url)
manager.add_request(request)
# 这里，区别于阻塞的调用的是visit_all方法，此地调用visit_iter方法
cursor_resp = manager.visit_iter()
async for resp in cursor_resp:
print('suc' if len(resp) > 0 else 'fail')
connection.close()