def run():
"""Parser by threads"""
goods = input_goods()
results = []
with ThreadPoolExecutor() as executor:
future_to_good = {
executor.submit(make_sync_request, good): good
for good in goods
}
# task = (executor.submit(make_sync_request, good) for good in goods)
for future in concurrent.futures.as_completed(future_to_good):
good = future_to_good[future]
try:
resp = future.result()
if resp is None:
print(f'No results found for {good}')
continue
results.extend(parse_html(resp.text))
except Exception as exc:
print('%r generated an exception: %s' % (good, exc))
csv_writer_to_file(results, file_name='thread_parser')
print(f'Done with {len(results)} results')
def run():
"""Sync parser"""
results = []
for good in input_goods():
resp = make_sync_request(good)
if resp is None:
continue
results.extend(parse_html(resp.text))
print(f'Done with {len(results)} results')
def run():
"""Sync parser"""
results = []
for good in input_goods():
resp = make_sync_request(good)
if resp is None:
continue
results.extend(parse_html(resp.text))
csv_writer_to_file(results, file_name='sync_parser')
print(f'Done with {len(results)} results')
def run():
"""Parser by processes"""
pool = mp.Pool(mp.cpu_count())
goods = input_goods()
results = []
async_results = [
pool.apply_async(make_sync_request, args=(good, )) for good in goods
] # is a list of pool.ApplyResult objects
for resp in (res.get() for res in async_results):
if resp is None:
continue
results.extend(parse_html(resp.text))
print(f'Done with {len(results)} results')
async def async_parser(goods):
tasks = []
# Fetch all responses within one Client session,
# keep connection alive for all requests.
async with ClientSession(headers=REQUEST_HEADERS,
cookies=PHARMACY_COOKIE) as session:
for good in goods:
task = asyncio.ensure_future(make_async_request(good, session))
tasks.append(task)
responses = await asyncio.gather(*tasks)
# you now have all response bodies in this variable
results = []
for resp in responses:
if resp is None:
continue
results.extend(parse_html(resp))
print(f'Done with {len(results)} results')