def test_process(self, response):
soup = self.links(spider, response)
# Look for forms to submit
if soup and url_data(response.url).get("use_forms", True):
for form in soup.findAll("form"):
spider_url_data = url_data(response.url)
form_id = form.get("id", form.get("name", None))
if form_id and spider_url_data.has_key("forms") and spider_url_data["forms"].has_key(form_id):
form_data = spider_url_data["forms"][form_id]
else:
form_data = spider_url_data
submit, query = build_form_query(self, form, response.url)
if form_data.has_key("data"):
for k, v in form_data["data"].items():
query[k] = v
if submit:
if form.get("method", "get").lower() == "get":
spider.spider_test(
urlparse.urljoin(
response.url, u"%s?%s" % (form["action"], x_www_form_urlencoded(query))
)
)
else:
self.links(
spider,
spider.agent.process(
urlparse.urljoin(response.url, form["action"]),
spider_url_data,
x_www_form_urlencoded(query),
),
)
def test_process(self, response):
soup = self.links(spider, response)
# Look for forms to submit
if soup and url_data(response.url).get('use_forms', True):
for form in soup.findAll('form'):
spider_url_data = url_data(response.url)
form_id = form.get('id', form.get('name', None))
if form_id \
and spider_url_data.has_key('forms') \
and spider_url_data['forms'].has_key(form_id):
form_data = spider_url_data['forms'][form_id]
else:
form_data = spider_url_data
submit, query = build_form_query(self, form, response.url)
if form_data.has_key('data'):
for k, v in form_data['data'].items():
query[k] = v
if submit:
if form.get('method', 'get').lower() == 'get':
spider.spider_test(
urlparse.urljoin(
response.url,
u'%s?%s' % (
form['action'],
x_www_form_urlencoded(query))))
else:
self.links(spider, spider.agent.process(
urlparse.urljoin(
response.url,
form['action']),
spider_url_data,
x_www_form_urlencoded(query)))
def process_form(spider, page_response, form, data={}, check_fn=lambda s, r: r.soup):
assert form, page_response.soup
submit, query = build_form_query(spider, form, page_response.url)
for k, v in data.items():
if v is None:
if query.has_key(k):
query.pop(k)
else:
query[k] = v
if form.get("method", "get").lower() == "get":
response = spider.agent.process(
urlparse.urljoin(page_response.url, u"%s?%s" % (form["action"], x_www_form_urlencoded(query)))
)
elif submit:
response = spider.agent.process(
urlparse.urljoin(page_response.url, form["action"]), {}, x_www_form_urlencoded(query)
)
else:
return None
check_fn(spider, response)
return response