def run(): """Parser by threads""" goods = input_goods() results = [] with ThreadPoolExecutor() as executor: future_to_good = { executor.submit(make_sync_request, good): good for good in goods } # task = (executor.submit(make_sync_request, good) for good in goods) for future in concurrent.futures.as_completed(future_to_good): good = future_to_good[future] try: resp = future.result() if resp is None: print(f'No results found for {good}') continue results.extend(parse_html(resp.text)) except Exception as exc: print('%r generated an exception: %s' % (good, exc)) csv_writer_to_file(results, file_name='thread_parser') print(f'Done with {len(results)} results')
def run(): """Sync parser""" results = [] for good in input_goods(): resp = make_sync_request(good) if resp is None: continue results.extend(parse_html(resp.text)) print(f'Done with {len(results)} results')
def run(): """Sync parser""" results = [] for good in input_goods(): resp = make_sync_request(good) if resp is None: continue results.extend(parse_html(resp.text)) csv_writer_to_file(results, file_name='sync_parser') print(f'Done with {len(results)} results')
def run(): """Parser by processes""" pool = mp.Pool(mp.cpu_count()) goods = input_goods() results = [] async_results = [ pool.apply_async(make_sync_request, args=(good, )) for good in goods ] # is a list of pool.ApplyResult objects for resp in (res.get() for res in async_results): if resp is None: continue results.extend(parse_html(resp.text)) print(f'Done with {len(results)} results')
def run(): """Parse site asynchronously""" goods = input_goods() loop = asyncio.get_event_loop() future = asyncio.ensure_future(async_parser(goods)) loop.run_until_complete(future)