def test_keyboard_interrupt(self):
m = Mock(side_effect=KeyboardInterrupt)
with patch('dirhunt.utils.confirm_close',
side_effect=KeyboardInterrupt) as mock_confirm_close:
with self.assertRaises(KeyboardInterrupt):
catch_keyboard_interrupt(m)()
mock_confirm_close.assert_called_once()
def test_delay_add_available(self):
sessions = Sessions(delay=1)
session = sessions.sessions[0]
with patch('dirhunt.sessions.threading.Timer') as m:
sessions.add_available(session)
m.assert_called_once_with(sessions.delay, sessions.availables.put,
[session])
m.return_value.start.assert_called_once()
def test_session_exception(self, req_mock):
req_mock.get(self.url, exc=requests.exceptions.ConnectTimeout)
crawler = self.get_crawler()
with patch('dirhunt.crawler_url.CrawlerUrl.close') as m:
crawler_url = CrawlerUrl(crawler, self.url)
self.assertEqual(crawler_url.start(), crawler_url)
self.assertEqual(crawler.current_processed_count, 1)
m.assert_called_once()
def test_error(self):
if sys.version_info < (3, ):
self.skipTest('Unsupported Mock in Python 2.7')
m = Mock(side_effect=DirHuntError)
with patch('sys.stderr.write') as mock_write:
catch(m)()
m.assert_called_once()
mock_write.assert_called_once()
def test_error(self):
if sys.version_info < (3, ):
self.skipTest('Unsupported Mock in Python 2.7')
m = Mock(side_effect=KeyError)
with patch('traceback.print_exc') as mock_print_exc:
with self.assertRaises(KeyError):
reraise_with_stack(m)()
m.assert_called_once()
mock_print_exc.assert_called_once()
def test_search(self, m1):
domain = 'domain'
urls = ['http://domain/path1', 'http://domain/path2']
with patch('dirhunt.sources.google.search',
side_effect=lambda x, stop: iter(urls)) as m2:
Google(lambda x: x, None).callback(domain)
m2.assert_called_once_with('site:{}'.format(domain),
stop=STOP_AFTER)
m1.assert_has_calls([call(url) for url in urls])
def test_session_read_exception(self):
crawler = self.get_crawler()
crawler.sessions = Mock()
crawler.sessions.get_session.return_value.get.return_value.status_code = 200
crawler.sessions.get_session.return_value.get.return_value.raw.read.side_effect = \
requests.exceptions.ConnectTimeout()
with patch('dirhunt.crawler_url.CrawlerUrl.close') as m:
crawler_url = CrawlerUrl(crawler, self.url)
self.assertEqual(crawler_url.start(), crawler_url)
self.assertEqual(crawler.current_processed_count, 1)
m.assert_called_once()
def test_failure(self, m1):
if sys.version_info < (3, ):
self.skipTest('Unsupported Mock usage in Python 2.7')
def search_iter(*args):
raise URLError('Test')
with patch('dirhunt.sources.google.search',
return_value=map(search_iter, [0])):
Google(lambda x: x, lambda x: x).callback('domain')
m1.assert_called_once()
def test_random_proxy_positive(self, m):
proxy_instance = Mock()
with patch('dirhunt.sessions.normalize_proxy',
return_value=proxy_instance):
sessions = Sessions()
session = sessions.sessions[0]
self.assertIs(session.proxy, proxy_instance)
session_mock = Mock()
session.session = session_mock
session.get(self.url)
proxy_instance.positive.assert_called_once()
def _test_random_proxy_negative(self, votes):
proxy_instance = Mock()
proxy_instance.get_updated_proxy.return_value.votes = votes
with patch('dirhunt.sessions.normalize_proxy',
return_value=proxy_instance):
sessions = Sessions()
session = sessions.sessions[0]
self.assertIs(session.proxy, proxy_instance)
session_mock = Mock(**{'get.side_effect': ProxyError})
session.session = session_mock
with self.assertRaises(ProxyError):
session.get(self.url)
return proxy_instance
def test_random_session(self):
sessions = Sessions()
sessions.availables.get()
with patch('dirhunt.sessions.random.choice') as m:
sessions.get_session()
m.assert_called_once()