def test_engine(self): so = get_html_doc('cmdextract', 'stackoverflow.com') query = ['du'] options = ['--engine', 'bing', '--no-cache'] with requests_mock.mock() as m: self._setup_request_mock(m, query, [so], search_engine='bing') main(query + options) self._check_output(contains='du')
def _test_engine(self, engine, expected_urls): e = get_engine(engine) req = e.get_search_request('test query') self.assertTrue('test+query' in req.url) doc = get_html_doc(TEST_DATA_DIR, '%s.com' % engine) urls = e.get_hits(doc) self.assertEqual([u.url for u in urls], expected_urls)
def _setup_request_mock(self, mock, query, search_results, search_engine='google'): engine_doc = get_html_doc('search_engines', search_engine + '.com') e = get_engine(search_engine) query_string, _ = searchcmd.get_query_string(query) url = e.get_search_request(query_string).url mock.get(url, content=engine_doc.body) urls = e.get_hits(engine_doc) for url, result in zip(urls, search_results): mock.get(url.url, content=result.body)
def test_verbose(self): so = get_html_doc('cmdextract', 'stackoverflow.com') query = ['du'] options = ['-v'] with requests_mock.mock() as m: self._setup_request_mock(m, query, [so]) main(query) non_verbose = self._get_current_stdout() self._truncate_stdout() with requests_mock.mock() as m: main(query + options) self.assertGreater(len(self._get_current_stdout()), len(non_verbose))
def test_cache(self): doc = get_html_doc('search_engines', 'google.com') docs = list(iter_html_docs('search_engines')) obj = {'single': doc, 'multi': docs} cache.store(obj, testing=True) from_cache = cache.get(testing=True) self.assertEqual(from_cache.keys(), obj.keys()) self.assertEqual(obj['single'].url.url, from_cache['single'].url.url) self.assertIsNone(cache.get(does_not_exist=True))
def test_cache(self): so = get_html_doc('cmdextract', 'stackoverflow.com') query = ['du'] options = ['--no-cache'] with requests_mock.mock() as m: self._setup_request_mock(m, query, [so]) main(query + options) self._check_output(contains='du') self._truncate_stdout() with requests_mock.mock() as m: main(query) self._check_output(contains='du')
def test_get(self): doc = get_html_doc('search_engines', 'google.com') req = download.Request(doc.url.url) with requests_mock.mock() as m: m.get(req.url, content=doc.body) resp = download.get(req) self.assertIsInstance(resp, download.HtmlDocument) self.assertEqual(resp.body, doc.body) with requests_mock.mock() as m: resp = download.get(req) self.assertIsInstance(resp, download.DownloadError)
def test_max_hits(self): so = get_html_doc('cmdextract', 'stackoverflow.com') query = ['du'] options = ['--max-hits', '1'] with requests_mock.mock() as m: self._setup_request_mock(m, query, [so]) main(query) nr_lines = len(self._get_current_stdout().split('\n')) self._truncate_stdout() with requests_mock.mock() as m: main(query + options) self.assertGreater( nr_lines, len(self._get_current_stdout().split('\n')))
def test_max_hits(self): so = get_html_doc('cmdextract', 'stackoverflow.com') query = ['du'] options = ['--max-hits', '1'] with requests_mock.mock() as m: self._setup_request_mock(m, query, [so]) main(query) nr_lines = len(self._get_current_stdout().split('\n')) self._truncate_stdout() with requests_mock.mock() as m: main(query + options) self.assertGreater(nr_lines, len(self._get_current_stdout().split('\n')))
def test_extract_commands(self): cmds = extract_commands(iter_html_docs(TEST_DATA_DIR)) self.assertEqual(set(cmds.commands.keys()), MERGED_COMMANDS) cmds = extract_commands(iter_html_docs(TEST_DATA_DIR), 'xargs') self.assertEqual(set(cmds.commands.keys()), set([ 'find /tmp -name "*.tmp" | xargs rm', u'find ./music -name "*.mp3" -print0 | xargs -0 ls', 'find . -name "*.sh" | xargs grep "ksh"', 'find /tmp -name "*.tmp" -print0 | xargs -0 rm', 'find . -name "*.sh" -print0 | xargs -0 -I {} mv {} ~/back.scripts', u'find ./work -print | xargs grep "profit"'])) cmds = extract_commands( get_html_doc(TEST_DATA_DIR, 'stackoverflow.com'), 'xargs') self.assertEqual(cmds.commands, {}) doc = HtmlDocument('http://stackoverflow.com', b'') doc.body = None cmds = extract_commands(doc) self.assertEqual(cmds.nr_docs, 0)