def _extract_cases_from_html(self, html):
"""Build list of data dictionaries, one dictionary per case (table row)."""
# Strip inconsistently placed <font> and <br>
# tags that make stable coverage almost impossible
etree.strip_tags(html, 'font', 'br')
path = '//table[@id="AutoNumber1"]//ul'
for ul in html.xpath(path):
preceding = ul.xpath('./preceding::*[1]')[0]
preceding_text = ' '.join(preceding.text_content().split()).strip(':')
# Skip sections that are marked to be published at future date
if preceding_text and not preceding_text.lower().endswith(' date'):
# Below will fail if they change up string format
date_string = preceding_text.split()[-1]
case_date = convert_date_string(date_string)
for element in ul.xpath('./li | ./a'):
if element.tag == 'li':
text = normalize_dashes(' '.join(element.text_content().split()))
if not text:
continue
anchor = element.xpath('.//a')[0]
elif element.tag == 'a':
# Malformed html, see connappct_example.html
anchor = element
glued = '%s %s' % (anchor.text_content(), anchor.tail)
text = normalize_dashes(' '.join(glued.split()))
self.cases.append({
'date': case_date,
'url': anchor.xpath('./@href')[0],
'docket': text.split('-')[0].replace('Concurrence', '').replace('Dissent', ''),
'name': text.split('-', 1)[1],
})
def _extract_cases_from_html(self, html):
"""Build list of data dictionaries, one dictionary per case (table row)."""
# Strip inconsistently placed <font> and <br>
# tags that make stable coverage almost impossible
etree.strip_tags(html, 'font', 'br')
for ul in html.xpath('//table[@id="AutoNumber1"]/tr[2]/td/table/tr/td//ul'):
preceding = ul.xpath('./preceding::*[1]')[0]
preceding_text = ' '.join(preceding.text_content().split()).strip(':')
if preceding_text and not preceding_text.lower().endswith('future date'):
# Below will fail if they change up strings or date formats
case_date = convert_date_string(preceding_text.split()[-1])
for element in ul.xpath('./li | ./a'):
if element.tag == 'li':
text = normalize_dashes(' '.join(element.text_content().split()))
if not text:
continue
anchor = element.xpath('.//a')[0]
elif element.tag == 'a':
# Malformed html, see connappct_example.html
anchor = element
glued = '%s %s' % (anchor.text_content(), anchor.tail)
text = normalize_dashes(' '.join(glued.split()))
self.cases.append({
'date': case_date,
'url': anchor.xpath('./@href')[0],
'docket': text.split('-')[0].replace('Concurrence', '').replace('Dissent', ''),
'name': text.split('-', 1)[1],
})
def _get_date_filed_is_approximate(self):
approximations = []
for element in self.html.xpath(self.date_string_path):
count = len(element.xpath(self.sub_opinion_path))
date_string = normalize_dashes(element.text_content())
approximation = True if '-' in date_string else False
approximations.extend([approximation] * count)
return approximations
def test_normalize_dashes(self):
tests = [
# copied from http://www.w3schools.com/charsets/ref_utf_punctuation.asp
u" this is –a test–", # en dash
u" this is —a test—", # em dash
u" this is ‐a test‐", # hyphen
u" this is ‑a test‑", # non-breaking hyphen
u" this is ‒a test‒", # figure dash
u" this is ―a test―", # horizontal bar
]
target = " this is -a test-"
for test in tests:
self.assertEqual(normalize_dashes(test), target)
def test_normalize_dashes(self):
tests = [
# copied from http://www.w3schools.com/charsets/ref_utf_punctuation.asp
u' this is –a test–', # en dash
u' this is —a test—', # em dash
u' this is ‐a test‐', # hyphen
u' this is ‑a test‑', # non-breaking hyphen
u' this is ‒a test‒', # figure dash
u' this is ―a test―', # horizontal bar
]
target = ' this is -a test-'
for test in tests:
self.assertEqual(normalize_dashes(test), target)
def _extract_cases_from_html(self, html):
"""Build list of data dictionaries, one dictionary per case (table row)."""
# Strip inconsistently placed <font> and <br>
# tags that make stable coverage almost impossible
etree.strip_tags(html, "font", "br")
path = '//table[@id="AutoNumber1"]//ul'
for ul in html.xpath(path):
preceding = ul.xpath("./preceding::*[1]")[0]
preceding_text = " ".join(
preceding.text_content().split()).strip(":")
# Skip sections that are marked to be published at future date
if preceding_text and not preceding_text.lower().endswith(" date"):
# Below will fail if they change up string format
date_string = preceding_text.split()[-1]
case_date = convert_date_string(date_string)
for element in ul.xpath("./li | ./a"):
if element.tag == "li":
text = normalize_dashes(" ".join(
element.text_content().split()))
if not text:
continue
anchor = element.xpath(".//a")[0]
elif element.tag == "a":
# Malformed html, see connappct_example.html
anchor = element
glued = "%s %s" % (anchor.text_content(), anchor.tail)
text = normalize_dashes(" ".join(glued.split()))
self.cases.append({
"date":
case_date,
"url":
anchor.xpath("./@href")[0],
"docket":
text.split("-")[0].replace("Concurrence",
"").replace("Dissent", ""),
"name":
text.split("-", 1)[1],
})
def sanitize_text(self, text):
"""Prevent non-standard characters and typos from breaking regex"""
return self.fix_court_year_id_typo(clean_string(
normalize_dashes(text)))
def sanitize_text(self, text):
"""Prevent non-standard characters and typos from breaking regex"""
return self.fix_court_year_id_typo(clean_string(normalize_dashes(text)))