def main(worker):
# 并发限制
sem = asyncio.Semaphore(worker)
loop = asyncio.get_event_loop()
tasks = [
asyncio.ensure_future(limit_req(url, sem))
for url in gen_url(20, False)
]
# 1.
loop.run_until_complete(run_tasks(tasks))
# 2.
# res = loop.run_until_complete(asyncio.gather(*tasks))
# for x in res:
# print(x)
# print('all completed:', res)
# 3.
# res = loop.run_until_complete(asyncio.wait(tasks))
# for task in tasks:
# print(task.result())
# print('all completed:', res)
# 4.
# res = loop.run_until_complete(asyncio.wait([limit_req(url, sem) for url in gen_url(20, False)]))
# for task in res[0]:
# print(task.result())
# print('all completed:', res)
loop.close()
async def aio_init(worker=100):
"""TCPConnector 连接池限制并发"""
# limit 限制同时连接数, 默认是100, limit=0 无限制
conn = aiohttp.TCPConnector(ssl=False, limit=worker)
async with aiohttp.ClientSession(connector=conn) as sess:
tasks = [asyncio.ensure_future(aio_get_url(url, sess)) for url in gen_url(20)]
await run_tasks(tasks)
async def aio_init_sem(worker):
"""Semaphore 限制并发"""
conn = aiohttp.TCPConnector(ssl=False)
async with aiohttp.ClientSession(connector=conn) as sess:
# 写在这儿
sem = asyncio.Semaphore(worker)
tasks = [asyncio.ensure_future(aio_get_url_sem(url, sem, sess)) for url in gen_url(20)]
await run_tasks(tasks)
def main(max_worker):
loop = asyncio.get_event_loop()
pool = ThreadPoolExecutor(max_worker)
tasks = [loop.run_in_executor(pool, get_url, *url) for url in gen_url(20, False)]
loop.run_until_complete(run_tasks(tasks))
loop.close()
def main(max_worker):
sess = Session()
loop = asyncio.get_event_loop()
pool = ThreadPoolExecutor(max_worker)
tasks = [loop.run_in_executor(pool, requests_get_url, url, sess) for url in gen_url(20)]
loop.run_until_complete(run_tasks(tasks))
loop.close()
def main(worker):
loop = asyncio.get_event_loop()
pool = ThreadPoolExecutor(worker)
tasks = [loop.run_in_executor(pool, get_url, *url) for url in gen_url(20, False)]
res = loop.run_until_complete(asyncio.wait(tasks))
loop.close()
for x in res[0]:
print(x.result())
print('result:', res)
def register(request):
if not request.user.is_authenticated():
form = RegisterForm(request.POST or None)
if form.is_valid():
userName = form.cleaned_data['username']
passWord = form.cleaned_data['password']
eMail = form.cleaned_data['email']
firstName = form.cleaned_data['first_name']
lastName = form.cleaned_data['last_name']
user = User.objects.create_user(
username=userName,
email=eMail,
password=passWord,
)
url = util.gen_url()
UserProfile.objects.create(user=user, url=url)
return HttpResponseRedirect('/')
return r2r(request, 'register.html', {'form': form})
return HttpResponseRedirect('/')
async def aio_init(sem):
conn = aiohttp.TCPConnector(ssl=False)
async with aiohttp.ClientSession(connector=conn) as sess:
tasks = [asyncio.ensure_future(aio_get_url(url, sem, sess)) for url in gen_url(20)]
await run_tasks(tasks)
def create_profile(sender, **kwargs):
if kwargs.get('created', False):
user = kwargs.get('instance')
url = util.gen_url()
UserProfile.objects.create(user=user, url=url)
def main():
for url in gen_url(20, False):
html = get_url(*url)
print('...' * 100, html)
def run_requests():
sess = requests.Session()
res = [requests_get_url(url, sess) for url in gen_url(20)]
print('total:', len(res))
async def aio_init():
async with aiohttp.ClientSession() as sess:
tasks = [aio_get_url(url, sess) for url in gen_url(20)]
return await asyncio.gather(*tasks)
def create_profile(sender, **kwargs):
if kwargs.get('created', False):
user = kwargs.get('instance')
url = util.gen_url()
UserProfile.objects.create(user=user, url=url)
def main(worker):
rs = [grequests.get(x) for x in gen_url(20)]
for r in grequests.imap(rs, size=worker):
r.encoding = 'utf-8'
print(r.text)
