def run():
"""Parser by threads"""
goods = input_goods()
results = []
with ThreadPoolExecutor() as executor:
future_to_good = {
executor.submit(make_sync_request, good): good
for good in goods
}
# task = (executor.submit(make_sync_request, good) for good in goods)
for future in concurrent.futures.as_completed(future_to_good):
good = future_to_good[future]
try:
resp = future.result()
if resp is None:
print(f'No results found for {good}')
continue
results.extend(parse_html(resp.text))
except Exception as exc:
print('%r generated an exception: %s' % (good, exc))
csv_writer_to_file(results, file_name='thread_parser')
print(f'Done with {len(results)} results')
def run():
"""Sync parser"""
results = []
for good in input_goods():
resp = make_sync_request(good)
if resp is None:
continue
results.extend(parse_html(resp.text))
print(f'Done with {len(results)} results')
def run():
"""Sync parser"""
results = []
for good in input_goods():
resp = make_sync_request(good)
if resp is None:
continue
results.extend(parse_html(resp.text))
csv_writer_to_file(results, file_name='sync_parser')
print(f'Done with {len(results)} results')
def run():
"""Parser by processes"""
pool = mp.Pool(mp.cpu_count())
goods = input_goods()
results = []
async_results = [
pool.apply_async(make_sync_request, args=(good, )) for good in goods
] # is a list of pool.ApplyResult objects
for resp in (res.get() for res in async_results):
if resp is None:
continue
results.extend(parse_html(resp.text))
print(f'Done with {len(results)} results')
def run():
"""Parse site asynchronously"""
goods = input_goods()
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(async_parser(goods))
loop.run_until_complete(future)