def import_mayer(case_path):
"""Open the file, get its contents, convert to XML and extract the meta data.
Return a document object for saving in the database
"""
#raw_text = open(case_path).read()
#clean_html_tree, complete_html_tree, clean_html_str, body_text = get_html_from_raw_text(raw_text)
tree = html.parse(case_path)
sha1 = hashlib.sha1(clean_html_str).hexdigest()
citations = get_citations_from_tree(complete_html_tree, case_path)
judges = get_judge(clean_html_tree, case_path)
court = get_court_object(clean_html_tree, citations, case_path, judges)
doc = Document(
source='L',
sha1=sha1,
html=clean_html_str, # we clear this field later, putting the value into html_lawbox.
date_filed=get_date_filed(clean_html_tree, citations=citations, case_path=case_path, court=court),
precedential_status=get_precedential_status(),
judges=judges,
download_url=case_path,
)
cite = Citation()
docket = Docket(
docket_number=get_docket_number(
clean_html_tree,
case_path=case_path,
court=court
),
case_name=get_case_name(complete_html_tree, case_path),
court=court,
)
# Necessary for dup_finder.
path = '//p/text()'
doc.body_text = ' '.join(clean_html_tree.xpath(path))
# Add the dict of citations to the object as its attributes.
citations_as_dict = map_citations_to_models(citations)
for k, v in citations_as_dict.iteritems():
setattr(cite, k, v)
doc.citation = cite
doc.docket = docket
return doc
def import_law_box_case(case_path):
"""Open the file, get its contents, convert to XML and extract the meta data.
Return a document object for saving in the database
"""
raw_text = open(case_path).read()
clean_html_tree, complete_html_tree, clean_html_str, body_text = get_html_from_raw_text(
raw_text)
sha1 = hashlib.sha1(clean_html_str).hexdigest()
citations = get_citations_from_tree(complete_html_tree, case_path)
judges = get_judge(clean_html_tree, case_path)
court = get_court_object(clean_html_tree, citations, case_path, judges)
doc = Document(
source='L',
sha1=sha1,
html=
clean_html_str, # we clear this field later, putting the value into html_lawbox.
date_filed=get_date_filed(clean_html_tree,
citations=citations,
case_path=case_path,
court=court),
precedential_status=get_precedential_status(),
judges=judges,
download_url=case_path,
)
cite = Citation(docket_number=get_docket_number(
clean_html_tree, case_path=case_path, court=court))
docket = Docket(
case_name=get_case_name(complete_html_tree, case_path),
court=court,
)
# Necessary for dup_finder.
path = '//p/text()'
doc.body_text = ' '.join(clean_html_tree.xpath(path))
# Add the dict of citations to the object as its attributes.
citations_as_dict = map_citations_to_models(citations)
for k, v in citations_as_dict.iteritems():
setattr(cite, k, v)
doc.citation = cite
doc.docket = docket
return doc
def setUp(self):
self.court = Court.objects.get(pk='test')
# create 3 documents with their citations and dockets
c1, c2, c3 = Citation(case_name=u"c1"), Citation(
case_name=u"c2"), Citation(case_name=u"c3")
c1.save(index=False)
c2.save(index=False)
c3.save(index=False)
docket1 = Docket(
case_name=u"c1",
court=self.court,
)
docket2 = Docket(
case_name=u"c2",
court=self.court,
)
docket3 = Docket(
case_name=u"c3",
court=self.court,
)
docket1.save()
docket2.save()
docket3.save()
d1, d2, d3 = Document(date_filed=date.today()), Document(
date_filed=date.today()), Document(date_filed=date.today())
d1.citation, d2.citation, d3.citation = c1, c2, c3
d1.docket, d2.docket, d3.docket = docket1, docket2, docket3
doc_list = [d1, d2, d3]
for d in doc_list:
d.citation.save(index=False)
d.save(index=False)
# create simple citing relation: 1 cites 2 and 3; 2 cites 3; 3 cites 1;
d1.cases_cited.add(d2.citation)
d2.citation_count += 1
d2.cases_cited.add(d3.citation)
d3.citation_count += 1
d3.cases_cited.add(d1.citation)
d1.citation_count += 1
d1.cases_cited.add(d3.citation)
d3.citation_count += 1
d1.save(index=False)
d2.save(index=False)
d3.save(index=False)
def setUp(self):
self.court = Court.objects.get(pk='test')
# create 3 documents with their citations and dockets
c1, c2, c3 = Citation(case_name=u"c1"), Citation(
case_name=u"c2"), Citation(case_name=u"c3")
c1.save(index=False)
c2.save(index=False)
c3.save(index=False)
docket1 = Docket(
case_name=u"c1",
court=self.court,
)
docket2 = Docket(
case_name=u"c2",
court=self.court,
)
docket3 = Docket(
case_name=u"c3",
court=self.court,
)
docket1.save()
docket2.save()
docket3.save()
d1, d2, d3 = Document(date_filed=date.today()), Document(
date_filed=date.today()), Document(date_filed=date.today())
d1.citation, d2.citation, d3.citation = c1, c2, c3
d1.docket, d2.docket, d3.docket = docket1, docket2, docket3
doc_list = [d1, d2, d3]
for d in doc_list:
d.citation.save(index=False)
d.save(index=False)
# create simple citing relation: 1 cites 2 and 3; 2 cites 3; 3 cites 1;
d1.cases_cited.add(d2.citation)
d2.citation_count += 1
d2.cases_cited.add(d3.citation)
d3.citation_count += 1
d3.cases_cited.add(d1.citation)
d1.citation_count += 1
d1.cases_cited.add(d3.citation)
d3.citation_count += 1
d1.save(index=False)
d2.save(index=False)
d3.save(index=False)
def import_resource_org_item(case_location):
"""Using the path to a case, import it, gathering all needed meta data.
Path is any valid URI that the requests library can handle.
"""
def get_file(location):
if location.startswith('/'):
with open(location) as f:
r = requests.Session()
r.content = f.read()
else:
r = requests.get(location)
return fromstring(r.content), get_clean_body_content(r.content)
# Get trees and text for the opinion itself and for the index page
# that links to it. Each has useful data.
case_tree, case_text = get_file(case_location)
vol_location = case_location.rsplit('/', 1)[-2] + '/index.html'
vol_tree, vol_text = get_file(vol_location)
html, blocked = anonymize(get_case_body(case_tree))
case_location_relative = case_location.rsplit('/', 1)[1]
case_name, status = get_case_name_and_status(
vol_tree, case_location_relative)
cite = Citation(
case_name=case_name,
docket_number=get_docket_number(case_location),
federal_cite_one=get_west_cite(vol_tree, case_location_relative),
)
docket = Docket(
court=Court.objects.get(pk=get_court_id(case_tree)),
case_name=case_name,
)
doc = Document(
date_filed=get_date_filed(vol_tree, case_location_relative),
source='R',
sha1=hashlib.sha1(case_text).hexdigest(),
citation=cite,
docket=docket,
download_url=case_location,
html=html,
precedential_status=status,
)
if blocked:
doc.blocked = True
docket.blocked = True
doc.date_blocked = datetime.date.today()
docket.date_blocked = datetime.date.today()
cite.save()
docket.save()
doc.docket = docket
doc.citation = cite
doc.save()
# Update the citation graph
from alert.citations.tasks import update_document_by_id
update_document_by_id(doc.pk)
return doc
def import_resource_org_item(case_location):
"""Using the path to a case, import it, gathering all needed meta data.
Path is any valid URI that the requests library can handle.
"""
def get_file(location):
if location.startswith('/'):
with open(location) as f:
r = requests.Session()
r.content = f.read()
else:
r = requests.get(location)
return fromstring(r.content), get_clean_body_content(r.content)
# Get trees and text for the opinion itself and for the index page
# that links to it. Each has useful data.
case_tree, case_text = get_file(case_location)
vol_location = case_location.rsplit('/', 1)[-2] + '/index.html'
vol_tree, vol_text = get_file(vol_location)
html, blocked = anonymize(get_case_body(case_tree))
case_location_relative = case_location.rsplit('/', 1)[1]
case_name, status = get_case_name_and_status(vol_tree,
case_location_relative)
cite = Citation(
case_name=case_name,
docket_number=get_docket_number(case_location),
federal_cite_one=get_west_cite(vol_tree, case_location_relative),
)
docket = Docket(
court=Court.objects.get(pk=get_court_id(case_tree)),
case_name=case_name,
)
doc = Document(
date_filed=get_date_filed(vol_tree, case_location_relative),
source='R',
sha1=hashlib.sha1(case_text).hexdigest(),
citation=cite,
docket=docket,
download_url=case_location,
html=html,
precedential_status=status,
)
if blocked:
doc.blocked = True
docket.blocked = True
doc.date_blocked = datetime.date.today()
docket.date_blocked = datetime.date.today()
cite.save()
docket.save()
doc.docket = docket
doc.citation = cite
doc.save()
# Update the citation graph
from alert.citations.tasks import update_document_by_id
update_document_by_id(doc.pk)
return doc
def test_pagerank_calculation(self):
"""Create a few Documents and fake citation relation among them, then
run the pagerank algorithm. Check whether this simple case can get the
correct result.
"""
# Set up some handy variables
self.court = Court.objects.get(pk='test')
# create 3 documents with their citations and dockets
c1, c2, c3 = Citation(case_name=u"c1"), Citation(
case_name=u"c2"), Citation(case_name=u"c3")
c1.save(index=False)
c2.save(index=False)
c3.save(index=False)
docket1 = Docket(
case_name=u"c1",
court=self.court,
)
docket2 = Docket(
case_name=u"c2",
court=self.court,
)
docket3 = Docket(
case_name=u"c3",
court=self.court,
)
d1, d2, d3 = Document(date_filed=date.today()), Document(
date_filed=date.today()), Document(date_filed=date.today())
d1.citation, d2.citation, d3.citation = c1, c2, c3
d1.docket, d2.docket, d3.docket = docket1, docket2, docket3
doc_list = [d1, d2, d3]
for d in doc_list:
d.citation.save(index=False)
d.save(index=False)
#create simple citing relation: 1 cites 2 and 3; 2 cites 3; 3 cites 1;
d1.cases_cited.add(d2.citation)
d2.citation_count += 1
d2.cases_cited.add(d3.citation)
d3.citation_count += 1
d3.cases_cited.add(d1.citation)
d1.citation_count += 1
d1.cases_cited.add(d3.citation)
d3.citation_count += 1
d1.save(index=False)
d2.save(index=False)
d3.save(index=False)
#calculate pagerank of these 3 document
comm = Command()
self.verbosity = 1
comm.do_pagerank(chown=False)
# read in the pagerank file, converting to a dict
pr_values_from_file = {}
with open(get_data_dir_location() + "external_pagerank") as f:
for line in f:
pk, value = line.split('=')
pr_values_from_file[pk] = float(value.strip())
# Verify that whether the answer is correct, based on calculations in
# Gephi
answers = {
'1': 0.387790,
'2': 0.214811,
'3': 0.397400,
}
for key, value in answers.iteritems():
self.assertTrue(
(abs(pr_values_from_file[key]) - value) < 0.0001,
msg="The answer for item %s was %s when it should have been "
"%s" % (
key,
pr_values_from_file[key],
answers[key],
))
