def _test_engine(self, engine, expected_urls):
        e = get_engine(engine)
        req = e.get_search_request('test query')
        self.assertTrue('test+query' in req.url)

        doc = get_html_doc(TEST_DATA_DIR, '%s.com' % engine)

        urls = e.get_hits(doc)
        self.assertEqual([u.url for u in urls], expected_urls)
Example #2
0
 def _setup_request_mock(self, mock, query, search_results,
                         search_engine='google'):
     engine_doc = get_html_doc('search_engines', search_engine + '.com')
     e = get_engine(search_engine)
     query_string, _ = searchcmd.get_query_string(query)
     url = e.get_search_request(query_string).url
     mock.get(url, content=engine_doc.body)
     urls = e.get_hits(engine_doc)
     for url, result in zip(urls, search_results):
         mock.get(url.url, content=result.body)
Example #3
0
def search(query_string=None, cmd=None, search_engine='google',
           max_download=5):
    engine = get_engine(search_engine)
    search_req = engine.get_search_request(query_string)
    search_result = get(search_req)
    if isinstance(search_result, DownloadError):
        raise SearchError('Failed search on {} ({})'.format(
            search_engine, search_result.status_code))
    urls = engine.get_hits(search_result)
    docs = iter_get([Request(u.url) for u in urls[:max_download]])

    return extract_commands(docs, base_commands=cmd)
 def test_non_existent(self):
     with self.assertRaises(ValueError):
         get_engine('non_existent')