def get_judges(self, node):
"""Parse out the judge string and then look it up in the DB"""
try:
s = self.case_details.xpath('%s/text()' % node)[0].strip()
except IndexError:
print " Couldn't get judge for node: %s" % node
return None, ''
else:
judge_names = find_judge_names(s)
judges = []
for judge_name in judge_names:
judges.append(
find_person(judge_name,
self.court.pk,
case_date=self.date_filed))
judges = [c for c in judges if c is not None]
if len(judges) == 0:
print " No judges found after lookup."
logger.info("No judge for: %s" %
((s, self.court.pk, self.date_filed), ))
return None, s
elif len(judges) == 1:
return judges[0], s
elif len(judges) > 1:
print " Too many judges found: %s" % len(judges)
return None, s
def get_judges(self, node):
"""Parse out the judge string and then look it up in the DB"""
try:
s = self.case_details.xpath('%s/text()' % node)[0].strip()
except IndexError:
print " Couldn't get judge for node: %s" % node
return None, ''
else:
judge_names = find_judge_names(s)
judges = []
for judge_name in judge_names:
judges.append(find_person(judge_name, self.court.pk,
case_date=self.date_filed))
judges = [c for c in judges if c is not None]
if len(judges) == 0:
print " No judges found after lookup."
logger.info("No judge for: %s" % (
(s, self.court.pk, self.date_filed),
))
return None, s
elif len(judges) == 1:
return judges[0], s
elif len(judges) > 1:
print " Too many judges found: %s" % len(judges)
return None, s
def assign_authors(testing=False):
clusters = OpinionCluster.objects.exclude(judges='')
for cluster in clusters:
print("Processing: %s" % cluster)
print(" Judge string: %s" % cluster.judges)
judges = find_judges(cluster.judges)
if len(judges) == 0:
continue
candidates = []
for judge in judges:
candidates.append(
find_person(judge,
cluster.docket.court_id,
case_date=cluster.date_filed))
candidates = [c for c in candidates if c is not None]
opinion = cluster.sub_opinions.all()[0]
if len(candidates) == 1:
opinion.author = candidates[0]
print ' Author assigned: ', candidates[0]
elif len(candidates) > 1:
opinion.panel = candidates
print ' Panel assigned:', candidates
else:
print ' No match.'
if not testing:
opinion.save()
matchcount = 0
panelcount = 0
zerocount = 0
for i, row in df.iterrows():
#if row.court_id not in cas:
# continue
if pd.isnull(row.judges):
continue
judges = find_judges(row.judges)
date_filed = dt.strptime(row.date_filed, "%Y-%m-%d")
candidates = []
for judge in judges:
candidates.append(find_person(judge, row.court_id, case_date=date_filed))
candidates = [c for c in candidates if c is not None]
if len(candidates) == 1:
author = candidates[0]
if len(candidates) > 1:
panel = candidates
if len(candidates) == 1:
matchcount += 1
if len(candidates) > 1:
panelcount += 1
if len(candidates) == 0:
zerocount += 1
def make_and_save(item, skipdupes=False, min_dates=None, testing=True):
"""Associates case data from `parse_opinions` with objects. Saves these
objects.
min_date: if not none, will skip cases after min_date
"""
date_filed = date_argued = date_reargued = date_reargument_denied = date_cert_granted = date_cert_denied = None
unknown_date = None
for date_cluster in item['dates']:
for date_info in date_cluster:
# check for any dates that clearly aren't dates
if date_info[1].year < 1600 or date_info[1].year > 2020:
continue
# check for untagged dates that will be assigned to date_filed
if date_info[0] is None:
date_filed = date_info[1]
continue
# try to figure out what type of date it is based on its tag string
if date_info[0] in FILED_TAGS:
date_filed = date_info[1]
elif date_info[0] in DECIDED_TAGS:
if not date_filed:
date_filed = date_info[1]
elif date_info[0] in ARGUED_TAGS:
date_argued = date_info[1]
elif date_info[0] in REARGUE_TAGS:
date_reargued = date_info[1]
elif date_info[0] in REARGUE_DENIED_TAGS:
date_reargument_denied = date_info[1]
elif date_info[0] in CERT_GRANTED_TAGS:
date_cert_granted = date_info[1]
elif date_info[0] in CERT_DENIED_TAGS:
date_cert_denied = date_info[1]
else:
unknown_date = date_info[1]
if date_info[0] not in UNKNOWN_TAGS:
print("\nFound unknown date tag '%s' with date '%s'.\n" %
date_info)
# the main date (used for date_filed in OpinionCluster) and panel dates
# (used for finding judges) are ordered in terms of which type of dates
# best reflect them
main_date = (date_filed or date_argued or date_reargued or
date_reargument_denied or unknown_date)
panel_date = (date_argued or date_reargued or date_reargument_denied or
date_filed or unknown_date)
if main_date is None:
raise Exception("Failed to get a date for " + item['file'])
if min_dates is not None:
if min_dates.get(item['court_id']) is not None:
if main_date >= min_dates[item['court_id']]:
print(main_date, 'after', min_dates[item['court_id']],
' -- skipping.')
return
docket = Docket(
source=Docket.COLUMBIA,
date_argued=date_argued,
date_reargued=date_reargued,
date_cert_granted=date_cert_granted,
date_cert_denied=date_cert_denied,
date_reargument_denied=date_reargument_denied,
court_id=item['court_id'],
case_name_short=item['case_name_short'] or '',
case_name=item['case_name'] or '',
case_name_full=item['case_name_full'] or '',
docket_number=item['docket'] or ''
)
# get citations in the form of, e.g. {'federal_cite_one': '1 U.S. 1', ...}
found_citations = []
for c in item['citations']:
found = get_citations(c)
if not found:
# if the docket number --is-- citation string, we're likely dealing
# with a somewhat common triplet of (docket number, date,
# jurisdiction), which isn't a citation at all (so there's no
# problem)
if item['docket']:
docket_no = item['docket'].lower()
if 'claim no.' in docket_no:
docket_no = docket_no.split('claim no.')[0]
for junk in DOCKET_JUNK:
docket_no = docket_no.replace(junk, '')
docket_no = docket_no.strip('.').strip()
if docket_no and docket_no in c.lower():
continue
# there are a trivial number of letters (except for months and a few
# trivial words) in the citation, then it's not a citation at all
non_trivial = c.lower()
for trivial in TRIVIAL_CITE_WORDS:
non_trivial = non_trivial.replace(trivial, '')
num_letters = sum(non_trivial.count(letter) for letter in string.lowercase)
if num_letters < 3:
continue
# if there is a string that's known to indicate a bad citation, then
# it's not a citation
if any(bad in c for bad in BAD_CITES):
continue
# otherwise, this is a problem
raise Exception("Failed to get a citation from the string '%s' in "
"court '%s' with docket '%s'." % (
c, item['court_id'], item['docket']
))
else:
found_citations.extend(found)
citations_map = map_citations_to_models(found_citations)
cluster = OpinionCluster(
judges=item.get('judges', '') or "",
precedential_status=('Unpublished' if item['unpublished'] else 'Published'),
date_filed=main_date,
case_name_short=item['case_name_short'] or '',
case_name=item['case_name'] or '',
case_name_full=item['case_name_full'] or '',
source='Z',
attorneys=item['attorneys'] or '',
posture=item['posture'] or '',
**citations_map
)
panel = [find_person(n, item['court_id'], case_date=panel_date) for n in
item['panel']]
panel = [x for x in panel if x is not None]
opinions = []
for i, opinion_info in enumerate(item['opinions']):
if opinion_info['author'] is None:
author = None
else:
author = find_person(opinion_info['author'], item['court_id'],
case_date=panel_date)
converted_text = convert_columbia_html(opinion_info['opinion'])
opinion_type = OPINION_TYPE_MAPPING[opinion_info['type']]
if opinion_type == '020lead' and i > 0:
opinion_type = '050addendum'
opinion = Opinion(
author=author,
per_curiam=opinion_info['per_curiam'],
type=opinion_type,
# type=OPINION_TYPE_MAPPING[opinion_info['type']],
html_columbia=converted_text,
sha1=opinion_info['sha1'],
local_path=opinion_info['local_path'],
)
joined_by = [find_person(n, item['court_id'], case_date=panel_date) for n in opinion_info['joining']]
joined_by = [x for x in joined_by if x is not None]
opinions.append((opinion, joined_by))
if min_dates is None:
# check to see if this is a duplicate
dups = find_dups(docket, cluster, panel, opinions)
if dups:
if skipdupes:
print('Duplicate. skipping.')
else:
raise Exception("Found %s duplicate(s)." % len(dups))
# save all the objects
if not testing:
try:
docket.save()
cluster.docket = docket
cluster.save(index=False)
for member in panel:
cluster.panel.add(member)
for opinion, joined_by in opinions:
opinion.cluster = cluster
opinion.save(index=False)
for joiner in joined_by:
opinion.joined_by.add(joiner)
if settings.DEBUG:
domain = "http://127.0.0.1:8000"
else:
domain = "https://www.courtlistener.com"
print("Created item at: %s%s" % (domain, cluster.get_absolute_url()))
except:
# if anything goes wrong, try to delete everything
try:
docket.delete()
except:
pass
raise
def make_and_save(item):
"""Associates case data from `parse_opinions` with objects. Saves these objects."""
date_filed = date_argued = date_reargued = date_reargument_denied = date_cert_granted = date_cert_denied = None
for date_cluster in item['dates']:
for date_info in date_cluster:
# check for any dates that clearly aren't dates
if date_info[1].year < 1600 or date_info[1].year > 2020:
continue
# check for untagged dates that will be assigned to date_filed
if date_info[0] is None:
date_filed = date_info[1]
continue
# try to figure out what type of date it is based on its tag string
if date_info[0] in FILED_TAGS:
date_filed = date_info[1]
elif date_info[0] in DECIDED_TAGS:
if not date_filed:
date_filed = date_info[1]
elif date_info[0] in ARGUED_TAGS:
date_argued = date_info[1]
elif date_info[0] in REARGUE_TAGS:
date_reargued = date_info[1]
elif date_info[0] in REARGUE_DENIED_TAGS:
date_reargument_denied = date_info[1]
elif date_info[0] in CERT_GRANTED_TAGS:
date_cert_granted = date_info[1]
elif date_info[0] in CERT_DENIED_TAGS:
date_cert_denied = date_info[1]
else:
print("Found unknown date tag '%s' with date '%s'." % date_info)
docket = Docket(
date_argued=date_argued
,date_reargued=date_reargued
,date_cert_granted=date_cert_granted
,date_cert_denied=date_cert_denied
,date_reargument_denied=date_reargument_denied
,court_id=item['court_id']
,case_name_short=item['case_name_short'] or ''
,case_name=item['case_name'] or ''
,case_name_full=item['case_name_full'] or ''
,docket_number=item['docket'] or ''
)
docket.save()
# get citations in the form of, e.g. {'federal_cite_one': '1 U.S. 1', ...}
found_citations = []
for c in item['citations']:
found = get_citations(c)
if not found:
raise Exception("Failed to get a citation from the string '%s'." % c)
elif len(found) > 1:
raise Exception("Got multiple citations from string '%s' when there should have been one." % c)
found_citations.append(found[0])
citations_map = map_citations_to_models(found_citations)
cluster = OpinionCluster(
docket=docket
,precedential_status=('Unpublished' if item['unpublished'] else 'Published')
,date_filed=date_filed
,case_name_short=item['case_name_short'] or ''
,case_name=item['case_name'] or ''
,case_name_full=item['case_name_full'] or ''
,source='Z'
,attorneys=item['attorneys'] or ''
,posture=item['posture'] or ''
,**citations_map
)
cluster.save()
if date_argued is not None:
paneldate = date_argued
else:
paneldate = date_filed
panel = [find_person(n, item['court_id'], paneldate) for n in item['panel']]
panel = [x for x in panel if x is not None]
for member in panel:
cluster.panel.add(member)
for opinion_info in item['opinions']:
if opinion_info['author'] is None:
author = None
else:
author = find_person(opinion_info['author'], item['court_id'], date_filed or date_argued)
opinion = Opinion(
cluster=cluster
,author=author
,type=OPINION_TYPE_MAPPING[opinion_info['type']]
,html_columbia=opinion_info['opinion']
)
opinion.save()
joined_by = [find_person(n, item['court_id'], paneldate) for n in opinion_info['joining']]
joined_by = [x for x in joined_by if x is not None]
for joiner in joined_by:
opinion.joined_by.add(joiner)
def make_and_save(item,
skipdupes=False,
min_dates=None,
start_dates=None,
testing=True):
"""Associates case data from `parse_opinions` with objects. Saves these
objects.
min_date: if not none, will skip cases after min_date
"""
date_filed = date_argued = date_reargued = date_reargument_denied = date_cert_granted = date_cert_denied = None
unknown_date = None
for date_cluster in item['dates']:
for date_info in date_cluster:
# check for any dates that clearly aren't dates
if date_info[1].year < 1600 or date_info[1].year > 2020:
continue
# check for untagged dates that will be assigned to date_filed
if date_info[0] is None:
date_filed = date_info[1]
continue
# try to figure out what type of date it is based on its tag string
if date_info[0] in FILED_TAGS:
date_filed = date_info[1]
elif date_info[0] in DECIDED_TAGS:
if not date_filed:
date_filed = date_info[1]
elif date_info[0] in ARGUED_TAGS:
date_argued = date_info[1]
elif date_info[0] in REARGUE_TAGS:
date_reargued = date_info[1]
elif date_info[0] in REARGUE_DENIED_TAGS:
date_reargument_denied = date_info[1]
elif date_info[0] in CERT_GRANTED_TAGS:
date_cert_granted = date_info[1]
elif date_info[0] in CERT_DENIED_TAGS:
date_cert_denied = date_info[1]
else:
unknown_date = date_info[1]
if date_info[0] not in UNKNOWN_TAGS:
print("\nFound unknown date tag '%s' with date '%s'.\n" %
date_info)
# the main date (used for date_filed in OpinionCluster) and panel dates
# (used for finding judges) are ordered in terms of which type of dates
# best reflect them
main_date = (date_filed or date_argued or date_reargued
or date_reargument_denied or unknown_date)
panel_date = (date_argued or date_reargued or date_reargument_denied
or date_filed or unknown_date)
if main_date is None:
raise Exception("Failed to get a date for " + item['file'])
# special rule for Kentucky
if item['court_id'] == 'kycourtapp' and main_date <= date(1975, 12, 31):
item['court_id'] = 'kycourtapphigh'
if min_dates is not None:
if min_dates.get(item['court_id']) is not None:
if main_date >= min_dates[item['court_id']]:
print(main_date, 'after', min_dates[item['court_id']],
' -- skipping.')
return
if start_dates is not None:
if start_dates.get(item['court_id']) is not None:
if main_date <= start_dates[item['court_id']]:
print(main_date, 'before court founding:',
start_dates[item['court_id']], ' -- skipping.')
return
docket = Docket(source=Docket.COLUMBIA,
date_argued=date_argued,
date_reargued=date_reargued,
date_cert_granted=date_cert_granted,
date_cert_denied=date_cert_denied,
date_reargument_denied=date_reargument_denied,
court_id=item['court_id'],
case_name_short=item['case_name_short'] or '',
case_name=item['case_name'] or '',
case_name_full=item['case_name_full'] or '',
docket_number=item['docket'] or '')
# get citations in the form of, e.g. {'federal_cite_one': '1 U.S. 1', ...}
found_citations = []
for c in item['citations']:
found = get_citations(c)
if not found:
# if the docket number --is-- citation string, we're likely dealing
# with a somewhat common triplet of (docket number, date,
# jurisdiction), which isn't a citation at all (so there's no
# problem)
if item['docket']:
docket_no = item['docket'].lower()
if 'claim no.' in docket_no:
docket_no = docket_no.split('claim no.')[0]
for junk in DOCKET_JUNK:
docket_no = docket_no.replace(junk, '')
docket_no = docket_no.strip('.').strip()
if docket_no and docket_no in c.lower():
continue
# there are a trivial number of letters (except for months and a few
# trivial words) in the citation, then it's not a citation at all
non_trivial = c.lower()
for trivial in TRIVIAL_CITE_WORDS:
non_trivial = non_trivial.replace(trivial, '')
num_letters = sum(
non_trivial.count(letter) for letter in string.lowercase)
if num_letters < 3:
continue
# if there is a string that's known to indicate a bad citation, then
# it's not a citation
if any(bad in c for bad in BAD_CITES):
continue
# otherwise, this is a problem
raise Exception("Failed to get a citation from the string '%s' in "
"court '%s' with docket '%s'." %
(c, item['court_id'], item['docket']))
else:
found_citations.extend(found)
citations_map = map_citations_to_models(found_citations)
cluster = OpinionCluster(
judges=item.get('judges', '') or "",
precedential_status=('Unpublished'
if item['unpublished'] else 'Published'),
date_filed=main_date,
case_name_short=item['case_name_short'] or '',
case_name=item['case_name'] or '',
case_name_full=item['case_name_full'] or '',
source='Z',
attorneys=item['attorneys'] or '',
posture=item['posture'] or '',
**citations_map)
panel = [
find_person(n, item['court_id'], case_date=panel_date)
for n in item['panel']
]
panel = [x for x in panel if x is not None]
opinions = []
for i, opinion_info in enumerate(item['opinions']):
if opinion_info['author'] is None:
author = None
else:
author = find_person(opinion_info['author'],
item['court_id'],
case_date=panel_date)
converted_text = convert_columbia_html(opinion_info['opinion'])
opinion_type = OPINION_TYPE_MAPPING[opinion_info['type']]
if opinion_type == '020lead' and i > 0:
opinion_type = '050addendum'
opinion = Opinion(
author=author,
per_curiam=opinion_info['per_curiam'],
type=opinion_type,
# type=OPINION_TYPE_MAPPING[opinion_info['type']],
html_columbia=converted_text,
sha1=opinion_info['sha1'],
local_path=opinion_info['local_path'],
)
joined_by = [
find_person(n, item['court_id'], case_date=panel_date)
for n in opinion_info['joining']
]
joined_by = [x for x in joined_by if x is not None]
opinions.append((opinion, joined_by))
if min_dates is None:
# check to see if this is a duplicate
dups = find_dups(docket, cluster)
if dups:
if skipdupes:
print('Duplicate. skipping.')
else:
raise Exception("Found %s duplicate(s)." % len(dups))
# save all the objects
if not testing:
try:
docket.save()
cluster.docket = docket
cluster.save(index=False)
for member in panel:
cluster.panel.add(member)
for opinion, joined_by in opinions:
opinion.cluster = cluster
opinion.save(index=False)
for joiner in joined_by:
opinion.joined_by.add(joiner)
if settings.DEBUG:
domain = "http://127.0.0.1:8000"
else:
domain = "https://www.courtlistener.com"
print("Created item at: %s%s" %
(domain, cluster.get_absolute_url()))
except:
# if anything goes wrong, try to delete everything
try:
docket.delete()
except:
pass
raise
matchcount = 0
panelcount = 0
zerocount = 0
for i, row in df.iterrows():
#if row.court_id not in cas:
# continue
if pd.isnull(row.judges):
continue
judges = find_judge_names(row.judges)
date_filed = dt.strptime(row.date_filed, "%Y-%m-%d")
candidates = []
for judge in judges:
candidates.append(
find_person(judge, row.court_id, case_date=date_filed))
candidates = [c for c in candidates if c is not None]
if len(candidates) == 1:
author = candidates[0]
print(author)
elif len(candidates) > 1:
panel = candidates
print(panel)
else:
print('No match.', row.judges)
if len(candidates) == 1:
matchcount += 1
if len(candidates) > 1:
def assign_authors(testing=False):
clusters = (OpinionCluster.objects.exclude(judges='').exclude(
docket__court__jurisdiction='FB').select_related(
'docket__court__id').only('date_filed', 'judges',
'docket__court_id'))
total = clusters.count()
i = 0
for cluster in clusters:
i += 1
print u"(%s/%s): Processing: %s, %s" % (i, total, cluster.pk,
cluster.date_filed)
#print u" Judge string: %s".encode('utf-8') % cluster.judges
judgestr = unidecode(cluster.judges)
print " Judge string: %s" % judgestr
if 'curiam' in judgestr.lower():
opinion = cluster.sub_opinions.all()[0]
opinion.per_curiam = True
print u' Per Curiam assigned.'
if not testing:
opinion.save(index=False)
continue
#judges = find_judge_names(cluster.judges)
judges = find_judge_names(judgestr)
if len(judges) == 0:
continue
candidates = []
for judge in judges:
candidates.append(
find_person(judge,
cluster.docket.court_id,
case_date=cluster.date_filed))
candidates = [c for c in candidates if c is not None]
if len(candidates) == 0:
# more than one judge token, but no DB matches, continue
print u' No match.'
continue
if len(candidates) > 1:
# more than one DB match, assign panel and continue
print u' Panel assigned: %s' % candidates
if not testing:
for candidate in candidates:
cluster.panel.add(candidate)
continue
# only one candidate, assign author
opinion = cluster.sub_opinions.all()[0]
if len(judges) == 1:
# one judge token, one DB match
opinion.author = candidates[0]
print ' Author assigned: %s' % unidecode(str(candidates[0]))
else:
# multiple judge tokens, one DB match
opinion.author = candidates[0]
print ' Author assigned: %s (with %d missing tokens)' % (
unidecode(str(candidates[0])), len(judges) - 1)
if not testing:
opinion.save(index=False)
def assign_authors(testing=False):
clusters = (OpinionCluster.objects
.exclude(judges='')
.exclude(docket__court__jurisdiction='FB')
.select_related('docket__court__id')
.only('date_filed', 'judges', 'docket__court_id'))
total = clusters.count()
i = 0
for cluster in clusters:
i += 1
print u"(%s/%s): Processing: %s, %s" % (i, total, cluster.pk,
cluster.date_filed)
#print u" Judge string: %s".encode('utf-8') % cluster.judges
judgestr = unidecode(cluster.judges)
print " Judge string: %s" % judgestr
if 'curiam' in judgestr.lower():
opinion = cluster.sub_opinions.all()[0]
opinion.per_curiam = True
print u' Per Curiam assigned.'
if not testing:
opinion.save(index=False)
continue
#judges = find_judge_names(cluster.judges)
judges = find_judge_names(judgestr)
if len(judges) == 0:
continue
candidates = []
for judge in judges:
candidates.append(find_person(judge,
cluster.docket.court_id,
case_date=cluster.date_filed))
candidates = [c for c in candidates if c is not None]
if len(candidates) == 0:
# more than one judge token, but no DB matches, continue
print u' No match.'
continue
if len(candidates) > 1:
# more than one DB match, assign panel and continue
print u' Panel assigned: %s' % candidates
if not testing:
for candidate in candidates:
cluster.panel.add(candidate)
continue
# only one candidate, assign author
opinion = cluster.sub_opinions.all()[0]
if len(judges) == 1:
# one judge token, one DB match
opinion.author = candidates[0]
print ' Author assigned: %s' % unidecode(str(candidates[0]))
else:
# multiple judge tokens, one DB match
opinion.author = candidates[0]
print ' Author assigned: %s (with %d missing tokens)' % (
unidecode(str(candidates[0])),
len(judges)-1
)
if not testing:
opinion.save(index=False)
def make_and_save(item,
skipdupes=False,
min_dates=None,
start_dates=None,
testing=True):
"""Associates case data from `parse_opinions` with objects. Saves these
objects.
min_date: if not none, will skip cases after min_date
"""
date_filed = (date_argued) = (
date_reargued
) = date_reargument_denied = date_cert_granted = date_cert_denied = None
unknown_date = None
for date_cluster in item["dates"]:
for date_info in date_cluster:
# check for any dates that clearly aren't dates
if date_info[1].year < 1600 or date_info[1].year > 2020:
continue
# check for untagged dates that will be assigned to date_filed
if date_info[0] is None:
date_filed = date_info[1]
continue
# try to figure out what type of date it is based on its tag string
if date_info[0] in FILED_TAGS:
date_filed = date_info[1]
elif date_info[0] in DECIDED_TAGS:
if not date_filed:
date_filed = date_info[1]
elif date_info[0] in ARGUED_TAGS:
date_argued = date_info[1]
elif date_info[0] in REARGUE_TAGS:
date_reargued = date_info[1]
elif date_info[0] in REARGUE_DENIED_TAGS:
date_reargument_denied = date_info[1]
elif date_info[0] in CERT_GRANTED_TAGS:
date_cert_granted = date_info[1]
elif date_info[0] in CERT_DENIED_TAGS:
date_cert_denied = date_info[1]
else:
unknown_date = date_info[1]
if date_info[0] not in UNKNOWN_TAGS:
print("\nFound unknown date tag '%s' with date '%s'.\n" %
date_info)
# the main date (used for date_filed in OpinionCluster) and panel dates
# (used for finding judges) are ordered in terms of which type of dates
# best reflect them
main_date = (date_filed or date_argued or date_reargued
or date_reargument_denied or unknown_date)
panel_date = (date_argued or date_reargued or date_reargument_denied
or date_filed or unknown_date)
if main_date is None:
raise Exception("Failed to get a date for " + item["file"])
# special rule for Kentucky
if item["court_id"] == "kycourtapp" and main_date <= date(1975, 12, 31):
item["court_id"] = "kycourtapphigh"
if min_dates is not None:
if min_dates.get(item["court_id"]) is not None:
if main_date >= min_dates[item["court_id"]]:
print(
main_date,
"after",
min_dates[item["court_id"]],
" -- skipping.",
)
return
if start_dates is not None:
if start_dates.get(item["court_id"]) is not None:
if main_date <= start_dates[item["court_id"]]:
print(
main_date,
"before court founding:",
start_dates[item["court_id"]],
" -- skipping.",
)
return
docket = Docket(
source=Docket.COLUMBIA,
date_argued=date_argued,
date_reargued=date_reargued,
date_cert_granted=date_cert_granted,
date_cert_denied=date_cert_denied,
date_reargument_denied=date_reargument_denied,
court_id=item["court_id"],
case_name_short=item["case_name_short"] or "",
case_name=item["case_name"] or "",
case_name_full=item["case_name_full"] or "",
docket_number=item["docket"] or "",
)
# get citation objects in a list for addition to the cluster
found_citations = []
for c in item["citations"]:
found = get_citations(c, clean=("html", "whitespace"))
if not found:
# if the docket number --is-- citation string, we're likely dealing
# with a somewhat common triplet of (docket number, date,
# jurisdiction), which isn't a citation at all (so there's no
# problem)
if item["docket"]:
docket_no = item["docket"].lower()
if "claim no." in docket_no:
docket_no = docket_no.split("claim no.")[0]
for junk in DOCKET_JUNK:
docket_no = docket_no.replace(junk, "")
docket_no = docket_no.strip(".").strip()
if docket_no and docket_no in c.lower():
continue
# there are a trivial number of letters (except for
# months and a few trivial words) in the citation,
# then it's not a citation at all
non_trivial = c.lower()
for trivial in TRIVIAL_CITE_WORDS:
non_trivial = non_trivial.replace(trivial, "")
num_letters = sum(
non_trivial.count(letter) for letter in string.lowercase)
if num_letters < 3:
continue
# if there is a string that's known to indicate
# a bad citation, then it's not a citation
if any(bad in c for bad in BAD_CITES):
continue
# otherwise, this is a problem
raise Exception("Failed to get a citation from the string '%s' in "
"court '%s' with docket '%s'." %
(c, item["court_id"], item["docket"]))
else:
found_citations.extend(found.to_model())
cluster = OpinionCluster(
judges=item.get("judges", "") or "",
precedential_status=("Unpublished"
if item["unpublished"] else "Published"),
date_filed=main_date,
case_name_short=item["case_name_short"] or "",
case_name=item["case_name"] or "",
case_name_full=item["case_name_full"] or "",
source="Z",
attorneys=item["attorneys"] or "",
posture=item["posture"] or "",
)
panel = [
find_person(n, item["court_id"], case_date=panel_date)
for n in item["panel"]
]
panel = [x for x in panel if x is not None]
opinions = []
for i, opinion_info in enumerate(item["opinions"]):
if opinion_info["author"] is None:
author = None
else:
author = find_person(opinion_info["author"],
item["court_id"],
case_date=panel_date)
converted_text = convert_columbia_html(opinion_info["opinion"])
opinion_type = OPINION_TYPE_MAPPING[opinion_info["type"]]
if opinion_type == Opinion.LEAD and i > 0:
opinion_type = Opinion.ADDENDUM
opinion = Opinion(
author=author,
per_curiam=opinion_info["per_curiam"],
type=opinion_type,
# type=OPINION_TYPE_MAPPING[opinion_info['type']],
html_columbia=converted_text,
sha1=opinion_info["sha1"],
# This is surely not updated for the new S3 world. If you're
# reading this, you'll need to update this code.
local_path=opinion_info["local_path"],
)
joined_by = [
find_person(n, item["court_id"], case_date=panel_date)
for n in opinion_info["joining"]
]
joined_by = [x for x in joined_by if x is not None]
opinions.append((opinion, joined_by))
if min_dates is None:
# check to see if this is a duplicate
dups = find_dups(docket, cluster)
if dups:
if skipdupes:
print("Duplicate. skipping.")
else:
raise Exception("Found %s duplicate(s)." % len(dups))
# save all the objects
if not testing:
try:
docket.save()
cluster.docket = docket
cluster.save(index=False)
for citation in found_citations:
citation.cluster = cluster
citation.save()
for member in panel:
cluster.panel.add(member)
for opinion, joined_by in opinions:
opinion.cluster = cluster
opinion.save(index=False)
for joiner in joined_by:
opinion.joined_by.add(joiner)
if settings.DEBUG:
domain = "http://127.0.0.1:8000"
else:
domain = "https://www.courtlistener.com"
print("Created item at: %s%s" %
(domain, cluster.get_absolute_url()))
except:
# if anything goes wrong, try to delete everything
try:
docket.delete()
except:
pass
raise
