def test_scrapingbee_js_snippet():
'''It should b64 encode the JavaScript code'''
req = ScrapingBeeRequest(
DEFAULT_URL,
params={
'js_snippet': 'window.scrollTo(0, document.body.scrollHeight);'
})
assert req.meta['scrapingbee']['params']['js_snippet'] == \
'd2luZG93LnNjcm9sbFRvKDAsIGRvY3VtZW50LmJvZHkuc2Nyb2xsSGVpZ2h0KTs='
def start_requests(self):
for url in self.start_urls:
yield ScrapingBeeRequest(
url,
params={
# 'render_js': False,
# 'block_ads': True,
# 'block_resources': False,
# 'js_snippet': JS_SNIPPET,
# 'premium_proxy': True,
# 'country_code': 'fr',
# 'return_page_source': True,
# 'wait': 3000,
# 'wait_for': '#swagger-ui',
},
headers={
# 'Accept-Language': 'En-US',
},
cookies={
# 'name_1': 'value_1',
})
def test_scrapingbee_request():
'''It should add the url to the request meta'''
req = ScrapingBeeRequest(DEFAULT_URL)
assert req.url == DEFAULT_URL
assert req.meta['scrapingbee']['params']['url'] == DEFAULT_URL
def test_scrapingbee_params():
'''It should add the parameters to the request meta'''
req = ScrapingBeeRequest(DEFAULT_URL, params={
'render_js': False,
})
assert req.meta['scrapingbee']['params']['render_js'] is False
def test_scrapingbee_request():
'''It should add the encoded url to the request meta'''
req = ScrapingBeeRequest('https://example.com?p=1')
assert req.meta['scrapingbee']['params']['url'] == \
'https%3A//example.com%3Fp%3D1'
def spb_req(url=DEFAULT_URL):
return ScrapingBeeRequest(url)
def get_spb_url(api_url, api_key, url):
encoded_url = ScrapingBeeRequest.process_url(url)
return f'{api_url}?api_key={api_key}&url={encoded_url}'