def main(worker): # 并发限制 sem = asyncio.Semaphore(worker) loop = asyncio.get_event_loop() tasks = [ asyncio.ensure_future(limit_req(url, sem)) for url in gen_url(20, False) ] # 1. loop.run_until_complete(run_tasks(tasks)) # 2. # res = loop.run_until_complete(asyncio.gather(*tasks)) # for x in res: # print(x) # print('all completed:', res) # 3. # res = loop.run_until_complete(asyncio.wait(tasks)) # for task in tasks: # print(task.result()) # print('all completed:', res) # 4. # res = loop.run_until_complete(asyncio.wait([limit_req(url, sem) for url in gen_url(20, False)])) # for task in res[0]: # print(task.result()) # print('all completed:', res) loop.close()
async def aio_init(worker=100): """TCPConnector 连接池限制并发""" # limit 限制同时连接数, 默认是100, limit=0 无限制 conn = aiohttp.TCPConnector(ssl=False, limit=worker) async with aiohttp.ClientSession(connector=conn) as sess: tasks = [asyncio.ensure_future(aio_get_url(url, sess)) for url in gen_url(20)] await run_tasks(tasks)
async def aio_init_sem(worker): """Semaphore 限制并发""" conn = aiohttp.TCPConnector(ssl=False) async with aiohttp.ClientSession(connector=conn) as sess: # 写在这儿 sem = asyncio.Semaphore(worker) tasks = [asyncio.ensure_future(aio_get_url_sem(url, sem, sess)) for url in gen_url(20)] await run_tasks(tasks)
def main(max_worker): loop = asyncio.get_event_loop() pool = ThreadPoolExecutor(max_worker) tasks = [loop.run_in_executor(pool, get_url, *url) for url in gen_url(20, False)] loop.run_until_complete(run_tasks(tasks)) loop.close()
def main(max_worker): sess = Session() loop = asyncio.get_event_loop() pool = ThreadPoolExecutor(max_worker) tasks = [loop.run_in_executor(pool, requests_get_url, url, sess) for url in gen_url(20)] loop.run_until_complete(run_tasks(tasks)) loop.close()
def main(worker): loop = asyncio.get_event_loop() pool = ThreadPoolExecutor(worker) tasks = [loop.run_in_executor(pool, get_url, *url) for url in gen_url(20, False)] res = loop.run_until_complete(asyncio.wait(tasks)) loop.close() for x in res[0]: print(x.result()) print('result:', res)
def register(request): if not request.user.is_authenticated(): form = RegisterForm(request.POST or None) if form.is_valid(): userName = form.cleaned_data['username'] passWord = form.cleaned_data['password'] eMail = form.cleaned_data['email'] firstName = form.cleaned_data['first_name'] lastName = form.cleaned_data['last_name'] user = User.objects.create_user( username=userName, email=eMail, password=passWord, ) url = util.gen_url() UserProfile.objects.create(user=user, url=url) return HttpResponseRedirect('/') return r2r(request, 'register.html', {'form': form}) return HttpResponseRedirect('/')
async def aio_init(sem): conn = aiohttp.TCPConnector(ssl=False) async with aiohttp.ClientSession(connector=conn) as sess: tasks = [asyncio.ensure_future(aio_get_url(url, sem, sess)) for url in gen_url(20)] await run_tasks(tasks)
def create_profile(sender, **kwargs): if kwargs.get('created', False): user = kwargs.get('instance') url = util.gen_url() UserProfile.objects.create(user=user, url=url)
def main(): for url in gen_url(20, False): html = get_url(*url) print('...' * 100, html)
def run_requests(): sess = requests.Session() res = [requests_get_url(url, sess) for url in gen_url(20)] print('total:', len(res))
async def aio_init(): async with aiohttp.ClientSession() as sess: tasks = [aio_get_url(url, sess) for url in gen_url(20)] return await asyncio.gather(*tasks)
def main(worker): rs = [grequests.get(x) for x in gen_url(20)] for r in grequests.imap(rs, size=worker): r.encoding = 'utf-8' print(r.text)