def parse(responses):
"""Using a list of responses, parse out useful information and return
it as a list of dicts.
"""
results = []
court_id = "Court not yet set."
for response in responses:
response.raise_for_status()
court_id = get_court_id_from_url(response.url)
set_response_encoding(response)
text = clean_html(response.text)
tree = get_html_parsed_text(text)
tree.rewrite_links(fix_links_in_lxml_tree, base_href=response.url)
opinion_count = int(
tree.xpath('//b[contains(text(), "Total number of '
'opinions reported")]')[0].tail)
if opinion_count == 0:
continue
rows = tree.xpath('(//table)[1]//tr[position() > 1]')
for row in rows:
if results:
# If we have results already, pass the previous result to
# the FreeOpinionRow object.
row = FreeOpinionRow(row, results[-1], court_id)
else:
row = FreeOpinionRow(row, {}, court_id)
results.append(row)
logger.info("Parsed %s results from written opinions report at %s" %
(len(results), court_id))
return results
def _post_process_response(self):
"""Cleanup to response object"""
self.tweak_response_object()
self.request["response"].raise_for_status()
set_response_encoding(self.request["response"])
def _post_process_response(self):
"""Cleanup to response object"""
self.tweak_response_object()
self.request['response'].raise_for_status()
set_response_encoding(self.request['response'])