def test_response(self):
self.assertRaises(AttributeError, google.response, None)
self.assertRaises(AttributeError, google.response, [])
self.assertRaises(AttributeError, google.response, '')
self.assertRaises(AttributeError, google.response, '[]')
response = self.mock_response('<html></html>')
self.assertEqual(google.response(response), [])
html = """
<div class="g">
<h3 class="r">
<a href="http://this.should.be.the.link/">
<b>This</b> is <b>the</b> title
</a>
</h3>
<div class="s">
<div class="kv" style="margin-bottom:2px">
<cite>
<b>test</b>.psychologies.com/
</cite>
<div class="_nBb">
<div style="display:inline" onclick="google.sham(this);" aria-expanded="false"
aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">
<span class="_O0">
</span>
</div>
<div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">
<ul>
<li class="_Ykb">
<a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent
.com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">
En cache
</a>
</li>
<li class="_Ykb">
<a class="_Zkb" href="/search?safe=off&q=related:test.psy.com/">
Pages similaires
</a>
</li>
</ul>
</div>
</div>
</div>
<span class="st">
This should be the content.
</span>
<br>
<div class="osl">
<a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">
Test Personnalité
</a> -
<a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">
Tests - Moi
</a> -
<a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">
Test Couple
</a>
-
<a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">
Test Amour
</a>
</div>
</div>
</div>
<div class="g">
<h3 class="r">
<a href="http://www.google.com/images?q=toto">
<b>This</b>
</a>
</h3>
</div>
<div class="g">
<h3 class="r">
<a href="http://www.google.com/search?q=toto">
<b>This</b> is
</a>
</h3>
</div>
<div class="g">
<h3 class="r">
<a href="€">
<b>This</b> is <b>the</b>
</a>
</h3>
</div>
<div class="g">
<h3 class="r">
<a href="/url?q=url">
<b>This</b> is <b>the</b>
</a>
</h3>
</div>
<p class="_Bmc" style="margin:3px 8px">
<a href="/search?num=20&safe=off&q=t&revid=1754833769&sa=X&ei=-&ved=">
suggestion <b>title</b>
</a>
</p>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[1]['suggestion'], 'suggestion title')
html = """
<li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
</li>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
response = mock.Mock(text='<html></html>',
url='https://sorry.google.com')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
response = mock.Mock(text='<html></html>',
url='https://www.google.com/sorry/IndexRedirect')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
def test_response(self):
self.assertRaises(AttributeError, google.response, None)
self.assertRaises(AttributeError, google.response, [])
self.assertRaises(AttributeError, google.response, '')
self.assertRaises(AttributeError, google.response, '[]')
response = self.mock_response('<html></html>')
self.assertEqual(google.response(response), [])
html = """
<div class="ZINbbc xpd O9g5cc uUPGi">
<div>
<div class="kCrYT">
<a href="/url?q=http://this.should.be.the.link/">
<div class="BNeawe">
<b>This</b> is <b>the</b> title
</div>
<div class="BNeawe">
http://website
</div>
</a>
</div>
<div class="kCrYT">
<div>
<div class="BNeawe">
<div>
<div class="BNeawe">
This should be the content.
</div>
</div>
</div>
</div>
</div>
</div>
</p>
<div class="ZINbbc xpd O9g5cc uUPGi">
<div>
<div class="kCrYT">
<span>
<div class="BNeawe">
Related searches
</div>
</span>
</div>
<div class="rVLSBd">
<a>
<div>
<div class="BNeawe">
suggestion title
</div>
</div>
</a>
</div>
</div>
</p>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[1]['suggestion'], 'suggestion title')
html = """
<li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
</li>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
response = mock.Mock(text='<html></html>',
url='https://sorry.google.com')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
response = mock.Mock(text='<html></html>',
url='https://www.google.com/sorry/IndexRedirect')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
def test_response(self):
self.assertRaises(AttributeError, google.response, None)
self.assertRaises(AttributeError, google.response, [])
self.assertRaises(AttributeError, google.response, '')
self.assertRaises(AttributeError, google.response, '[]')
response = self.mock_response('<html></html>')
self.assertEqual(google.response(response), [])
html = """
<li class="g">
<h3 class="r">
<a href="http://this.should.be.the.link/">
<b>This</b> is <b>the</b> title
</a>
</h3>
<div class="s">
<div class="kv" style="margin-bottom:2px">
<cite>
<b>test</b>.psychologies.com/
</cite>
<div class="_nBb">
<div style="display:inline" onclick="google.sham(this);" aria-expanded="false"
aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">
<span class="_O0">
</span>
</div>
<div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">
<ul>
<li class="_Ykb">
<a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent
.com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">
En cache
</a>
</li>
<li class="_Ykb">
<a class="_Zkb" href="/search?safe=off&q=related:test.psy.com/">
Pages similaires
</a>
</li>
</ul>
</div>
</div>
</div>
<span class="st">
This should be the content.
</span>
<br>
<div class="osl">
<a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">
Test Personnalité
</a> -
<a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">
Tests - Moi
</a> -
<a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">
Test Couple
</a>
-
<a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">
Test Amour
</a>
</div>
</div>
</li>
<li class="g">
<h3 class="r">
<a href="http://www.google.com/images?q=toto">
<b>This</b>
</a>
</h3>
</li>
<li class="g">
<h3 class="r">
<a href="http://www.google.com/search?q=toto">
<b>This</b> is
</a>
</h3>
</li>
<li class="g">
<h3 class="r">
<a href="€">
<b>This</b> is <b>the</b>
</a>
</h3>
</li>
<li class="g">
<h3 class="r">
<a href="/url?q=url">
<b>This</b> is <b>the</b>
</a>
</h3>
</li>
<p class="_Bmc" style="margin:3px 8px">
<a href="/search?num=20&safe=off&q=t&revid=1754833769&sa=X&ei=-&ved=">
suggestion <b>title</b>
</a>
</p>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[1]['suggestion'], 'suggestion title')
html = """
<li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
</li>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
response = mock.Mock(text='<html></html>', url='https://sorry.google.com')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
import requests
from json import dumps
from searx.engines import google
from searx.search import default_request_params
request_params = default_request_params()
# Possible params
# request_params['headers']['User-Agent'] = ''
# request_params['category'] = ''
request_params['pageno'] = 1
request_params['language'] = 'en_us'
request_params['time_range'] = ''
params = google.request(argv[1], request_params)
request_args = dict(
headers=request_params['headers'],
cookies=request_params['cookies'],
)
if request_params['method'] == 'GET':
req = requests.get
else:
req = requests.post
request_args['data'] = request_params['data']
resp = req(request_params['url'], **request_args)
resp.search_params = request_params
print(dumps(google.response(resp)))
