def test_engine(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['--engine', 'bing', '--no-cache']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so], search_engine='bing')
main(query + options)
self._check_output(contains='du')
def test_engine(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['--engine', 'bing', '--no-cache']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so], search_engine='bing')
main(query + options)
self._check_output(contains='du')
def _test_engine(self, engine, expected_urls):
e = get_engine(engine)
req = e.get_search_request('test query')
self.assertTrue('test+query' in req.url)
doc = get_html_doc(TEST_DATA_DIR, '%s.com' % engine)
urls = e.get_hits(doc)
self.assertEqual([u.url for u in urls], expected_urls)
def _test_engine(self, engine, expected_urls):
e = get_engine(engine)
req = e.get_search_request('test query')
self.assertTrue('test+query' in req.url)
doc = get_html_doc(TEST_DATA_DIR, '%s.com' % engine)
urls = e.get_hits(doc)
self.assertEqual([u.url for u in urls], expected_urls)
def _setup_request_mock(self, mock, query, search_results,
search_engine='google'):
engine_doc = get_html_doc('search_engines', search_engine + '.com')
e = get_engine(search_engine)
query_string, _ = searchcmd.get_query_string(query)
url = e.get_search_request(query_string).url
mock.get(url, content=engine_doc.body)
urls = e.get_hits(engine_doc)
for url, result in zip(urls, search_results):
mock.get(url.url, content=result.body)
def test_verbose(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['-v']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so])
main(query)
non_verbose = self._get_current_stdout()
self._truncate_stdout()
with requests_mock.mock() as m:
main(query + options)
self.assertGreater(len(self._get_current_stdout()), len(non_verbose))
def test_verbose(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['-v']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so])
main(query)
non_verbose = self._get_current_stdout()
self._truncate_stdout()
with requests_mock.mock() as m:
main(query + options)
self.assertGreater(len(self._get_current_stdout()), len(non_verbose))
def test_cache(self):
doc = get_html_doc('search_engines', 'google.com')
docs = list(iter_html_docs('search_engines'))
obj = {'single': doc, 'multi': docs}
cache.store(obj, testing=True)
from_cache = cache.get(testing=True)
self.assertEqual(from_cache.keys(), obj.keys())
self.assertEqual(obj['single'].url.url, from_cache['single'].url.url)
self.assertIsNone(cache.get(does_not_exist=True))
def test_cache(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['--no-cache']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so])
main(query + options)
self._check_output(contains='du')
self._truncate_stdout()
with requests_mock.mock() as m:
main(query)
self._check_output(contains='du')
def test_get(self):
doc = get_html_doc('search_engines', 'google.com')
req = download.Request(doc.url.url)
with requests_mock.mock() as m:
m.get(req.url, content=doc.body)
resp = download.get(req)
self.assertIsInstance(resp, download.HtmlDocument)
self.assertEqual(resp.body, doc.body)
with requests_mock.mock() as m:
resp = download.get(req)
self.assertIsInstance(resp, download.DownloadError)
def test_get(self):
doc = get_html_doc('search_engines', 'google.com')
req = download.Request(doc.url.url)
with requests_mock.mock() as m:
m.get(req.url, content=doc.body)
resp = download.get(req)
self.assertIsInstance(resp, download.HtmlDocument)
self.assertEqual(resp.body, doc.body)
with requests_mock.mock() as m:
resp = download.get(req)
self.assertIsInstance(resp, download.DownloadError)
def test_cache(self):
doc = get_html_doc('search_engines', 'google.com')
docs = list(iter_html_docs('search_engines'))
obj = {'single': doc, 'multi': docs}
cache.store(obj, testing=True)
from_cache = cache.get(testing=True)
self.assertEqual(from_cache.keys(), obj.keys())
self.assertEqual(obj['single'].url.url, from_cache['single'].url.url)
self.assertIsNone(cache.get(does_not_exist=True))
def test_cache(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['--no-cache']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so])
main(query + options)
self._check_output(contains='du')
self._truncate_stdout()
with requests_mock.mock() as m:
main(query)
self._check_output(contains='du')
def test_max_hits(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['--max-hits', '1']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so])
main(query)
nr_lines = len(self._get_current_stdout().split('\n'))
self._truncate_stdout()
with requests_mock.mock() as m:
main(query + options)
self.assertGreater(
nr_lines, len(self._get_current_stdout().split('\n')))
def test_max_hits(self):
so = get_html_doc('cmdextract', 'stackoverflow.com')
query = ['du']
options = ['--max-hits', '1']
with requests_mock.mock() as m:
self._setup_request_mock(m, query, [so])
main(query)
nr_lines = len(self._get_current_stdout().split('\n'))
self._truncate_stdout()
with requests_mock.mock() as m:
main(query + options)
self.assertGreater(nr_lines,
len(self._get_current_stdout().split('\n')))
def _setup_request_mock(self,
mock,
query,
search_results,
search_engine='google'):
engine_doc = get_html_doc('search_engines', search_engine + '.com')
e = get_engine(search_engine)
query_string, _ = searchcmd.get_query_string(query)
url = e.get_search_request(query_string).url
mock.get(url, content=engine_doc.body)
urls = e.get_hits(engine_doc)
for url, result in zip(urls, search_results):
mock.get(url.url, content=result.body)
def test_extract_commands(self):
cmds = extract_commands(iter_html_docs(TEST_DATA_DIR))
self.assertEqual(set(cmds.commands.keys()), MERGED_COMMANDS)
cmds = extract_commands(iter_html_docs(TEST_DATA_DIR), 'xargs')
self.assertEqual(set(cmds.commands.keys()), set([
'find /tmp -name "*.tmp" | xargs rm',
u'find ./music -name "*.mp3" -print0 | xargs -0 ls',
'find . -name "*.sh" | xargs grep "ksh"',
'find /tmp -name "*.tmp" -print0 | xargs -0 rm',
'find . -name "*.sh" -print0 | xargs -0 -I {} mv {} ~/back.scripts',
u'find ./work -print | xargs grep "profit"']))
cmds = extract_commands(
get_html_doc(TEST_DATA_DIR, 'stackoverflow.com'), 'xargs')
self.assertEqual(cmds.commands, {})
doc = HtmlDocument('http://stackoverflow.com', b'')
doc.body = None
cmds = extract_commands(doc)
self.assertEqual(cmds.nr_docs, 0)
