def parse(responses): """Using a list of responses, parse out useful information and return it as a list of dicts. """ results = [] court_id = "Court not yet set." for response in responses: response.raise_for_status() court_id = get_court_id_from_url(response.url) set_response_encoding(response) text = clean_html(response.text) tree = get_html_parsed_text(text) tree.rewrite_links(fix_links_in_lxml_tree, base_href=response.url) opinion_count = int( tree.xpath('//b[contains(text(), "Total number of ' 'opinions reported")]')[0].tail) if opinion_count == 0: continue rows = tree.xpath('(//table)[1]//tr[position() > 1]') for row in rows: if results: # If we have results already, pass the previous result to # the FreeOpinionRow object. row = FreeOpinionRow(row, results[-1], court_id) else: row = FreeOpinionRow(row, {}, court_id) results.append(row) logger.info("Parsed %s results from written opinions report at %s" % (len(results), court_id)) return results
def _post_process_response(self): """Cleanup to response object""" self.tweak_response_object() self.request["response"].raise_for_status() set_response_encoding(self.request["response"])
def _post_process_response(self): """Cleanup to response object""" self.tweak_response_object() self.request['response'].raise_for_status() set_response_encoding(self.request['response'])