def test_generic_form_requests_with_file_field(self):
name = "ebay2"
spider = self.smanager.create(name)
generic_form_request = list(spider.start_requests())[0]
self.assertEqual(generic_form_request.url, 'file://tmp/test_params.txt')
response = HtmlResponse(url='file://tmp/test_params.txt', body=open(join(_PATH, "data", "test_params.txt")).read())
response.request = generic_form_request
requests = list(generic_form_request.callback(response))
request_list = [request_to_dict(req, spider) for req in requests]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {u'xpath': u"//form[@name='adv_search_from']", u'form_url': u'http://*****:*****@name='_nkw']", 'file_values': ['Cars', 'Boats'], u'type': u'inurl', u'value': u'file://tmp/test_params.txt'}, {u'type': u'inurl', u'name': u'_nkw2', u'value': u'file://tmp/test_params.txt'}, {u'xpath': u".//*[@name='_in_kw']", u'type': u'iterate'}]}, 'headers': {}, 'url': u'file://tmp/test_params.txt', 'dont_filter': True, 'priority': 0, 'callback': 'parse_field_url_page', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
generic_form_request = requests[0]
self.assertEqual(generic_form_request.url, 'file://tmp/test_params.txt')
response = HtmlResponse(url='file://tmp/test_params.txt', body=open(join(_PATH, "data", "test_params.txt")).read())
response.request = generic_form_request
requests = list(generic_form_request.callback(response))
request_list = [request_to_dict(req, spider) for req in requests]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {u'xpath': u"//form[@name='adv_search_from']", u'fields': [{u'xpath': u".//*[@name='_nkw']", 'file_values': ['Cars', 'Boats'], u'type': u'inurl', u'value': u'file://tmp/test_params.txt'}, {'file_values': ['Cars', 'Boats'], u'type': u'inurl', u'name': u'_nkw2', u'value': u'file://tmp/test_params.txt'}, {u'xpath': u".//*[@name='_in_kw']", u'type': u'iterate'}], u'type': u'form', 'field_index': 1}, 'headers': {}, 'url': u'http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc', 'dont_filter': True, 'priority': 0, 'callback': 'parse_form_page', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
generic_form_request = requests[0]
self.assertEqual(generic_form_request.url, 'http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc')
response = HtmlResponse(url="http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc", body=open(join(_PATH, "data", "ebay_advanced_search.html")).read())
response.request = generic_form_request
request_list = [request_to_dict(req, spider)
for req in generic_form_request.callback(response)]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc', 'dont_filter': True, 'priority': 0, 'callback': 'parse', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
def test_login_requests(self):
name = "pinterest.com"
spider = self.smanager.create(name)
login_request = list(spider.start_requests())[0]
response = HtmlResponse(url="https://pinterest.com/login/", body=open(join(_PATH, "data", "pinterest.html")).read())
response.request = login_request
form_request = login_request.callback(response)
expected = {'_encoding': 'utf-8',
'body': 'email=test&password=testpass&csrfmiddlewaretoken=nLZy3NMzhTswZvweHJ4KVmq9UjzaZGn3&_ch=ecnwmar2',
'callback': 'after_login',
'cookies': {},
'dont_filter': True,
'errback': None,
'headers': {'Content-Type': ['application/x-www-form-urlencoded']},
'meta': {},
'method': 'POST',
'priority': 0,
'url': u'https://pinterest.com/login/?next=%2F'}
self.assertEqual(request_to_dict(form_request, spider), expected)
# simulate a simple response to login post from which extract a link
response = HtmlResponse(url="http://pinterest.com/", body="<html><body><a href='http://pinterest.com/categories'></body></html>")
result = list(spider.after_login(response))
self.assertEqual([r.url for r in result], ['http://pinterest.com/categories', 'http://pinterest.com/popular/'])
def test_spider_crawls_links(spider, scrape_request, html_headers,
mock_html_twolinks):
"""Ensure spider always picks up relevant links to HTML pages"""
# Use only 1 user agent for easier counting
ua = factories.BatchUserAgentFactory.build(ua_string='Firefox / 11.0')
spider.batch_user_agents = [ua]
# Generate a mock response based on html containing two links
mock_response = HtmlResponse(url='http://test:12345',
body=mock_html_twolinks,
encoding='utf-8')
mock_response.request = scrape_request
mock_response.headers = html_headers
mock_response.meta['user_agent'] = ua
mock_response.meta['sitescan'] = factories.SiteScanFactory()
mock_response.status = 200
mock_response.flags = []
# Call spider on the mock response
pipeline_generator = spider.parse(mock_response)
# We should have two new requests and one MarkupItem
sites_expected = set([
mock_response.url + '/link1.html',
mock_response.url + '/link2.html',
])
sites_collected = []
for elem in pipeline_generator:
if isinstance(elem, Request):
sites_collected.append(elem.url)
else:
assert isinstance(elem, MarkupItem)
assert sites_expected == set(sites_collected)
def test_spider_crawls_links(spider, scrape_request, html_headers,
mock_html_twolinks):
"""Ensure spider always picks up relevant links to HTML pages"""
# Use only 1 user agent for easier counting
ua = factories.BatchUserAgentFactory.build(ua_string='Firefox / 11.0')
spider.batch_user_agents = [ua]
# Generate a mock response based on html containing two links
mock_response = HtmlResponse(url='http://test:12345',
body=mock_html_twolinks,
encoding='utf-8')
mock_response.request = scrape_request
mock_response.headers = html_headers
mock_response.meta['user_agent'] = ua
mock_response.meta['sitescan'] = factories.SiteScanFactory()
mock_response.status = 200
mock_response.flags = []
# Call spider on the mock response
pipeline_generator = spider.parse(mock_response)
# We should have two new requests and one MarkupItem
sites_expected = set([
mock_response.url + '/link1.html',
mock_response.url + '/link2.html',
])
sites_collected = []
for elem in pipeline_generator:
if isinstance(elem, Request):
sites_collected.append(elem.url)
else:
assert isinstance(elem, MarkupItem)
assert sites_expected == set(sites_collected)
def ad_hoc_test():
import os
from pprint import pprint
file_path = os.path.join(_research_dir,
'20191017/Amazon.com_ wooden toys.html')
print(os.path.abspath(file_path))
html = None
with open(file_path, 'r') as content_file:
html = content_file.read()
query = 'test query'
location = {'country': 'US', 'state': 'WA'}
spider = AmzLocalSearchWebSpider(query)
spider.crawler = None
response = HtmlResponse('http://test', status=200, body=html)
response.request = spider._make_first_request(
spider._args(query=query,
location=location,
max_pages=3,
tracking_id='test'))
result = spider.parse_response(response)
pprint(list(result))
def get_response(url, meta={}):
url = canonicalize_url(url)
r = requests.get(url)
res = r.text
final_url = r.url
to_encoding = 'utf-8'
response = HtmlResponse(url=final_url, body=res, encoding=to_encoding)
response.request = Request(url, meta=meta)
return response
def test_generic_form_requests_with_spider_args(self):
name = "ebay3"
args = {'search_string': 'Cars'}
spider = self.smanager.create(name, **args)
generic_form_request = list(spider.start_requests())[0]
response = HtmlResponse(url="http://www.ebay.com/sch/ebayadvsearch/?rt=nc", body=open(join(_PATH, "data", "ebay_advanced_search.html")).read())
response.request = generic_form_request
request_list = [request_to_dict(req, spider)
for req in generic_form_request.callback(response)]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=', 'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None}, {'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url': u'http://www.ebay.com/sch/ebayadvsearch/?rt=nc', 'dont_filter': True, 'priority': 0, 'callback': 'parse', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
def get_response(url, meta={}):
url = canonicalize_url(url)
r = requests.get(url)
res = r.text
final_url = r.url
to_encoding = 'utf-8'
response = HtmlResponse(url=final_url, body=res, encoding=to_encoding)
response.request = Request(url, meta=meta)
return response
def _get_url(url, request_kwargs={}):
'''Returns a scrapy.html.HtmlResponse with the contents of the received
url.
Note that the session is kept intact among multiple calls to this
method (i.e. cookies are passed over).
'''
response = betamax_session.get(url)
scrapy_response = HtmlResponse(
url=str(response.url),
body=response.content,
)
scrapy_response.request = Request(url, **request_kwargs)
return scrapy_response
def _get_url(url, request_kwargs={}):
'''Returns a scrapy.html.HtmlResponse with the contents of the received
url.
Note that the session is kept intact among multiple calls to this
method (i.e. cookies are passed over).
'''
response = betamax_session.get(url)
scrapy_response = HtmlResponse(
url=str(response.url),
body=response.content,
)
scrapy_response.request = Request(url, **request_kwargs)
return scrapy_response
def _get_url(url, request_kwargs={}):
'''Returns a scrapy.html.HtmlResponse with the contents of the received
url.
Note that the session is kept intact among multiple calls to this
method (i.e. cookies are passed over).
We also don't verify SSL certificates, because Takeda's certificate is
invalid. If they become valid, we can resume verifying the
certificates.
'''
response = betamax_session.get(url, verify=False)
scrapy_response = HtmlResponse(
url=str(response.url),
body=response.content,
)
scrapy_response.request = Request(url, **request_kwargs)
return scrapy_response
def parse_review_js(self, response):
for line in response.body.split('\n'):
if line.startswith('var materials='):
body = line.lstrip('var materials=').rstrip(',')
break
try:
body = eval(body)
except:
logging.error('Failed to parse: ' + repr(response.body))
# Emulate "normal" HTML response
body = ('<html><body>' + '%s' +
'</body></html>') % (body['BVRRSourceID'].replace('\\/', '/'))
response2 = HtmlResponse(url=response.url, body=body)
response2.request = response.request
for x in self.parse_review(response2):
yield x
def test_start_url_matcher(self):
url = 'http://example.org'
spider = self.spider_factory(start_urls=[url])
response = HtmlResponse(url)
rule = spider._rulesman.get_rule_from_response(response)
self.failUnless(isinstance(rule.matcher, UrlListMatcher))
response = HtmlResponse(url + '/item.html')
rule = spider._rulesman.get_rule_from_response(response)
self.failUnless(rule is None)
# TODO: remove this block
# in previous version get_rule returns rule from response.request
response.request = Request(url)
rule = spider._rulesman.get_rule_from_response(response.request)
self.failUnless(isinstance(rule.matcher, UrlListMatcher))
self.failUnlessEqual(rule.follow, True)
def test_parse_drug_details_or_overview_generates_new_request_if_redirected_to_search_page(self):
url = 'http://www.accessdata.fda.gov/scripts/cder/drugsatfda/index.cfm?fuseaction=Search.Search_Drug_Name'
meta = {
'original_url': 'http://www.accessdata.fda.gov/somewhere.cfm',
'original_cookies': {
'foo': 'bar',
},
}
mock_response = HtmlResponse(url=url)
mock_response.request = Request(url, meta=meta)
with mock.patch('random.random', return_value='random_cookiejar'):
spider = Spider()
request = spider.parse_drug_details_or_overview(mock_response)
assert request.url == meta['original_url']
assert request.cookies == meta['original_cookies']
assert request.dont_filter
assert request.callback == spider.parse_drug_details_or_overview
assert request.meta['cookiejar'] == 'random_cookiejar'
def _get_page(self, url=None, data=None, req=None): # {{{
""" Get response from url or request
if req is given, url/data is not used
if data is given, issue POST
"""
if not req:
req = Request(url, method="POST", body=data) if data else Request(url)
r = urlopen(req.url, req.body) if req.method == "POST" else urlopen(req.url)
contentType = r.info().getheader("Content-Type", None)
headers = {}
if contentType:
headers["Content-Type"] = contentType
response = HtmlResponse(req.url, body=r.read(), headers=headers)
response.request = req
return response
def test_start_url_matcher(self):
url = 'http://example.org'
spider = self.spider_factory(start_urls=[url])
response = HtmlResponse(url)
rule = spider._rulesman.get_rule_from_response(response)
self.failUnless(isinstance(rule.matcher, UrlListMatcher))
response = HtmlResponse(url + '/item.html')
rule = spider._rulesman.get_rule_from_response(response)
self.failUnless(rule is None)
# TODO: remove this block
# in previous version get_rule returns rule from response.request
response.request = Request(url)
rule = spider._rulesman.get_rule_from_response(response.request)
self.failUnless(isinstance(rule.matcher, UrlListMatcher))
self.failUnlessEqual(rule.follow, True)
def test_parse_drug_details_or_overview_generates_new_request_if_redirected_to_search_page(
self):
url = 'http://www.accessdata.fda.gov/scripts/cder/drugsatfda/index.cfm?fuseaction=Search.Search_Drug_Name'
meta = {
'original_url': 'http://www.accessdata.fda.gov/somewhere.cfm',
'original_cookies': {
'foo': 'bar',
},
}
mock_response = HtmlResponse(url=url)
mock_response.request = Request(url, meta=meta)
with mock.patch('random.random', return_value='random_cookiejar'):
spider = Spider()
request = spider.parse_drug_details_or_overview(mock_response)
assert request.url == meta['original_url']
assert request.cookies == meta['original_cookies']
assert request.dont_filter
assert request.callback == spider.parse_drug_details_or_overview
assert request.meta['cookiejar'] == 'random_cookiejar'
def parse_second(self, response):
"""
Called when navigating to the next search result page.
This handles json(?) response and turns it into
regular html response that can be handled by parse()
to avoid duplication of fragile pagination code
"""
try:
body = json.loads(response.body)
except:
logging.error('Failed to parse: ' + repr(response.body))
# Emulate "normal" HTML response
body = (
'<html><body>%s' + '%s' +
'<div id="dump_paging"><a href=""><span>NEXT</span></a></div></body></html>'
) % (body[9], body[2])
response2 = HtmlResponse(url=response.url,
body=str(body.encode('utf-8')))
response2.request = response.request
for x in self.parse_first(response2):
yield x
def make_response(url, body=None, meta=None, req_args=None, **resp_args):
def content_type(url):
ext = os.path.splitext(url)[1].strip('.')
if ext in ['html']:
return 'text/html'
elif ext in ['jpg', 'png']:
return 'image/jpeg'
else:
return 'text/html'
headers = {'Content-Type': content_type(url)}
if body:
#body = get_testdata(body) or get_testurl(body) or body
body = get_testdata(body) or body
response = HtmlResponse(url, headers=headers, body=body, **resp_args)
else:
response = Response(url, headers=headers, **resp_args)
request = Request(url, meta=meta, **(req_args or {}))
response.request = request
return response
def test_generic_form_requests_with_file_field(self):
name = "ebay2"
spider = self.smanager.create(name)
generic_form_request = list(spider.start_requests())[0]
self.assertEqual(generic_form_request.url, 'file://tmp/test_params.txt')
response = HtmlResponse(url='file://tmp/test_params.txt', body=open(join(_PATH, "data", "test_params.txt")).read())
response.request = generic_form_request
requests = list(generic_form_request.callback(response))
request_list = [request_to_dict(req, spider) for req in requests]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {u'xpath': u"//form[@name='adv_search_from']", u'form_url': u'http://www.ebay.com/sch/ebayadvsearch/?rt=nc', u'type': u'form', 'field_index': 1, u'fields': [{u'xpath': u".//*[@name='_nkw']", 'file_values': ['Cars', 'Boats'], u'type': u'inurl', u'value': u'file://tmp/test_params.txt'}, {u'type': u'inurl', u'name': u'_nkw2', u'value': u'file://tmp/test_params.txt'}, {u'xpath': u".//*[@name='_in_kw']", u'type': u'iterate'}]}, 'headers': {}, 'url': u'file://tmp/test_params.txt', 'dont_filter': True, 'priority': 0, 'callback': 'parse_field_url_page', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
generic_form_request = requests[0]
self.assertEqual(generic_form_request.url, 'file://tmp/test_params.txt')
response = HtmlResponse(url='file://tmp/test_params.txt', body=open(join(_PATH, "data", "test_params.txt")).read())
response.request = generic_form_request
requests = list(generic_form_request.callback(response))
request_list = [request_to_dict(req, spider) for req in requests]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {u'xpath': u"//form[@name='adv_search_from']", u'fields': [{u'xpath': u".//*[@name='_nkw']", 'file_values': ['Cars', 'Boats'], u'type': u'inurl', u'value': u'file://tmp/test_params.txt'}, {'file_values': ['Cars', 'Boats'], u'type': u'inurl', u'name': u'_nkw2', u'value': u'file://tmp/test_params.txt'}, {u'xpath': u".//*[@name='_in_kw']", u'type': u'iterate'}], u'type': u'form', 'field_index': 1}, 'headers': {}, 'url': u'http://www.ebay.com/sch/ebayadvsearch/?rt=nc', 'dont_filter': True, 'priority': 0, 'callback': 'parse_form_page', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
generic_form_request = requests[0]
self.assertEqual(generic_form_request.url, 'http://www.ebay.com/sch/ebayadvsearch/?rt=nc')
response = HtmlResponse(url="http://www.ebay.com/sch/ebayadvsearch/?rt=nc", body=open(join(_PATH, "data", "ebay_advanced_search.html")).read())
response.request = generic_form_request
request_list = [request_to_dict(req, spider)
for req in generic_form_request.callback(response)]
expected = [{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Cars&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {}, 'url':
u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Cars&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=1&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=2&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=3&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/i.html?_adv=1&_ex_kw=&_ftrv=1&_ftrt=901&_sabdlo=&_sabdhi=&_sop=12&_samihi=&_ipg=50&_salic=1&_sasl=&_udlo=&_okw=&_nkw2=Boats&_fsradio=%26LH_SpecificSeller%3D1&_udhi=&_in_kw=4&_nkw=Boats&_sacat=0&_oexkw=&_dmd=1&_saslop=1&_samilow=',
'dont_filter': True, 'priority': 0, 'callback': 'after_form_page', 'method': 'GET', 'errback': None},
{'body': '', '_encoding': 'utf-8', 'cookies': {}, 'meta': {}, 'headers': {},
'url': u'http://www.ebay.com/sch/ebayadvsearch/?rt=nc',
'dont_filter': True, 'priority': 0, 'callback': 'parse', 'method': 'GET', 'errback': None}]
self.assertEqual(request_list, expected)
def create_response(self, url, filename):
filename = os.path.join(self.directory, 'sample_data', filename)
response = HtmlResponse(url, body=open(filename).read())
response.request = Request(url)
return response
def parse_product_reviews(self, response):
for line in response.body.split('\n'):
if line.startswith('var materials='):
body = line.lstrip('var materials=').rstrip(',')
break
try:
body = eval(body)
except:
logging.error('Failed to parse: ' + repr(response.body))
body = ''
# Emulate "normal" HTML response
if body:
body = ('<html><body>' + '%s' + '</body></html>') % (
body['BVRRSourceID'].replace('\\/', '/'))
response2 = HtmlResponse(url=response.url, body=body)
response2.request = response.request
hxs = HtmlXPathSelector(response2) if body else None
base_url = self.get_base_url(response)
product = response.meta['product']
product['metadata'].setdefault('reviews', [])
box_spec = self.PRODUCT_REVIEW_BOX or {}
review_hxs = xpath_select(
hxs, box_spec.get('xpath')
) if 'xpath' in box_spec and box_spec.get('xpath') != "." else hxs
for review_box in review_hxs:
loader = ReviewLoader(item=Review(),
selector=hxs,
date_format=self.PRODUCT_REVIEW_DATE_FORMAT)
loader.add_value('url', urlparse.urljoin(base_url, response.url))
# review full text
full_text_specs = box_spec.get('full_text', []) if hasattr(
box_spec.get('full_text', []),
'append') else [box_spec['full_text']]
full_text_parts = []
for xpath in full_text_specs:
items = xpath_select(
review_box,
xpath).extract() if not callable(xpath) else [xpath(hxs)]
if any(items):
item_text = self.REVIEW_TEXT_JOIN.join([
e.replace(u'\xa0', u' ').strip(self.REVIEW_TEXT_STRIP)
for e in items
])
full_text_parts.append(item_text)
review_text = self.REVIEW_PARAGRAPH_JOIN.join(full_text_parts)
loader.add_value('full_text', review_text)
if box_spec.get('date'):
date = review_box.select(
box_spec.get('date')).extract() if not callable(
box_spec.get('date')) else [
box_spec['date'](review_box)
]
loader.add_value('date', date[0] if date else None)
if box_spec.get('rating'):
rating_text = review_box.select(
box_spec.get('rating')).extract() if not callable(
box_spec.get('rating')) else [
box_spec['rating'](review_box)
]
loader.add_value('rating',
rating_text[0] if rating_text else None)
review = loader.load_item()
if review.get('full_text') or review.get('date'):
product['metadata']['reviews'].append(review)
next_page = xpath_select(hxs, box_spec.get('next_url')).extract() if (
box_spec.get('next_url') and not callable(box_spec['next_url'])
) else [box_spec['next_url'](response, hxs)] if callable(
box_spec.get('next_url')) else None
next_page_url = urlparse.urljoin(
base_url, next_page[0]) if any(next_page) else None
if not next_page_url or next_page_url in self.visited_urls or not review_hxs:
yield self.clean_product(product)
else:
self.visited_urls.add(next_page_url)
yield Request(url=next_page_url,
meta=dict(**response.meta),
callback=self.parse_product_reviews)
