def _test_engine(self, engine, expected_urls): e = get_engine(engine) req = e.get_search_request('test query') self.assertTrue('test+query' in req.url) doc = get_html_doc(TEST_DATA_DIR, '%s.com' % engine) urls = e.get_hits(doc) self.assertEqual([u.url for u in urls], expected_urls)
def _setup_request_mock(self, mock, query, search_results, search_engine='google'): engine_doc = get_html_doc('search_engines', search_engine + '.com') e = get_engine(search_engine) query_string, _ = searchcmd.get_query_string(query) url = e.get_search_request(query_string).url mock.get(url, content=engine_doc.body) urls = e.get_hits(engine_doc) for url, result in zip(urls, search_results): mock.get(url.url, content=result.body)
def search(query_string=None, cmd=None, search_engine='google', max_download=5): engine = get_engine(search_engine) search_req = engine.get_search_request(query_string) search_result = get(search_req) if isinstance(search_result, DownloadError): raise SearchError('Failed search on {} ({})'.format( search_engine, search_result.status_code)) urls = engine.get_hits(search_result) docs = iter_get([Request(u.url) for u in urls[:max_download]]) return extract_commands(docs, base_commands=cmd)
def test_non_existent(self): with self.assertRaises(ValueError): get_engine('non_existent')