def test_attorney_role_normalization(self):
"""Can we normalize the attorney roles into a small number of roles?"""
pairs = [{
'q': '(Inactive)',
'a': {'role': Role.INACTIVE, 'date_action': None},
}, {
'q': 'ATTORNEY IN SEALED GROUP',
'a': {'role': Role.ATTORNEY_IN_SEALED_GROUP, 'date_action': None},
}, {
'q': 'ATTORNEY TO BE NOTICED',
'a': {'role': Role.ATTORNEY_TO_BE_NOTICED, 'date_action': None},
}, {
'q': 'Bar Status: ACTIVE',
'a': {'role': None, 'date_action': None},
}, {
'q': 'DISBARRED 02/19/2010',
'a': {'role': Role.DISBARRED,
'date_action': datetime.date(2010, 2, 19)},
}, {
'q': 'Designation: ADR Pro Bono Limited Scope Counsel',
'a': {'role': None, 'date_action': None},
}, {
'q': 'LEAD ATTORNEY',
'a': {'role': Role.ATTORNEY_LEAD, 'date_action': None},
}, {
'q': 'PRO HAC VICE',
'a': {'role': Role.PRO_HAC_VICE, 'date_action': None},
}, {
'q': 'SELF- TERMINATED: 01/14/2013',
'a': {'role': Role.SELF_TERMINATED,
'date_action': datetime.date(2013, 1, 14)},
}, {
'q': 'SUSPENDED 01/22/2016',
'a': {'role': Role.SUSPENDED,
'date_action': datetime.date(2016, 1, 22)},
}, {
'q': 'TERMINATED: 01/01/2007',
'a': {'role': Role.TERMINATED,
'date_action': datetime.date(2007, 1, 1)},
}]
for pair in pairs:
print("Normalizing PACER role of '%s' to '%s'..." %
(pair['q'], pair['a']), end='')
result = normalize_attorney_role(pair['q'])
self.assertEqual(result, pair['a'])
print('✓')
with self.assertRaises(ValueError):
normalize_attorney_role('this is an unknown role')
def test_attorney_role_normalization(self):
"""Can we normalize the attorney roles into a small number of roles?"""
pairs = [{
'q': '(Inactive)',
'a': {'role': Role.INACTIVE, 'date_action': None},
}, {
'q': 'ATTORNEY IN SEALED GROUP',
'a': {'role': Role.ATTORNEY_IN_SEALED_GROUP, 'date_action': None},
}, {
'q': 'ATTORNEY TO BE NOTICED',
'a': {'role': Role.ATTORNEY_TO_BE_NOTICED, 'date_action': None},
}, {
'q': 'Bar Status: ACTIVE',
'a': {'role': None, 'date_action': None},
}, {
'q': 'DISBARRED 02/19/2010',
'a': {'role': Role.DISBARRED,
'date_action': datetime.date(2010, 2, 19)},
}, {
'q': 'Designation: ADR Pro Bono Limited Scope Counsel',
'a': {'role': None, 'date_action': None},
}, {
'q': 'LEAD ATTORNEY',
'a': {'role': Role.ATTORNEY_LEAD, 'date_action': None},
}, {
'q': 'PRO HAC VICE',
'a': {'role': Role.PRO_HAC_VICE, 'date_action': None},
}, {
'q': 'SELF- TERMINATED: 01/14/2013',
'a': {'role': Role.SELF_TERMINATED,
'date_action': datetime.date(2013, 1, 14)},
}, {
'q': 'SUSPENDED 01/22/2016',
'a': {'role': Role.SUSPENDED,
'date_action': datetime.date(2016, 1, 22)},
}, {
'q': 'TERMINATED: 01/01/2007',
'a': {'role': Role.TERMINATED,
'date_action': datetime.date(2007, 1, 1)},
}]
for pair in pairs:
print("Normalizing PACER role of '%s' to '%s'..." %
(pair['q'], pair['a']), end='')
result = normalize_attorney_role(pair['q'])
self.assertEqual(result, pair['a'])
print('✓')
with self.assertRaises(ValueError):
normalize_attorney_role('this is an unknown role')
def normalize_attorney_roles(parties):
"""Clean up the attorney roles for all parties.
We do this fairly early in the process because we need to know if
there are any terminated attorneys before we can start
adding/removing content to/from the database. By normalizing
early, we ensure we have good data for that sniffing.
A party might be input with an attorney such as:
{
'name': 'William H. Narwold',
'contact': ("1 Corporate Center\n",
"20 Church Street\n",
"17th Floor\n",
"Hartford, CT 06103\n",
"860-882-1676\n",
"Fax: 860-882-1682\n",
"Email: [email protected]"),
'roles': ['LEAD ATTORNEY',
'TERMINATED: 03/12/2013'],
}
The role attribute will be cleaned up to be:
'roles': [{
'role': Role.ATTORNEY_LEAD,
'date_action': None,
'role_raw': 'LEAD ATTORNEY',
}, {
'role': Role.TERMINATED,
'date_action': date(2013, 3, 12),
'role_raw': 'TERMINATED: 03/12/2013',
}
:param parties: The parties dict from Juriscraper.
:returns None; editing happens in place.
"""
for party in parties:
for atty in party.get('attorneys', []):
roles = [normalize_attorney_role(r) for r in atty['roles']]
roles = remove_duplicate_dicts(roles)
atty['roles'] = roles
def add_attorney(atty, p, d):
"""Add/update an attorney.
Given an attorney node, and a party and a docket object, add the attorney
to the database or link the attorney to the new docket. Also add/update the
attorney organization, and the attorney's role in the case.
:param atty: A dict representing an attorney, as provided by Juriscraper.
:param p: A Party object
:param d: A Docket object
:return: None if there's an error, or an Attorney object if not.
"""
newest_docket_date = max([dt for dt in [d.date_filed, d.date_terminated,
d.date_last_filing] if dt])
atty_org_info, atty_info = normalize_attorney_contact(
atty['contact'],
fallback_name=atty['name'],
)
try:
q = Q()
fields = {
('phone', atty_info['phone']),
('fax', atty_info['fax']),
('email', atty_info['email']),
('contact_raw', atty['contact']),
('organizations__lookup_key', atty_org_info.get('lookup_key')),
}
for field, lookup in fields:
if lookup:
q |= Q(**{field: lookup})
a, created = Attorney.objects.filter(
Q(name=atty['name']) & q,
).distinct().get_or_create(
defaults={
'name': atty['name'],
'date_sourced': newest_docket_date,
'contact_raw': atty['contact'],
},
)
except Attorney.MultipleObjectsReturned:
logger.info("Got too many results for attorney: '%s'. Punting." % atty)
return None
# Associate the attorney with an org and update their contact info.
if atty['contact']:
if atty_org_info:
logger.info("Adding organization information to '%s': '%s'" %
(atty['name'], atty_org_info))
try:
org = AttorneyOrganization.objects.get(
lookup_key=atty_org_info['lookup_key'],
)
except AttorneyOrganization.DoesNotExist:
try:
org = AttorneyOrganization.objects.create(**atty_org_info)
except IntegrityError:
# Race condition. Item was created after get. Try again.
org = AttorneyOrganization.objects.get(
lookup_key=atty_org_info['lookup_key'],
)
# Add the attorney to the organization
AttorneyOrganizationAssociation.objects.get_or_create(
attorney=a,
attorney_organization=org,
docket=d,
)
docket_info_is_newer = (a.date_sourced <= newest_docket_date)
if atty_info and docket_info_is_newer:
logger.info("Updating atty info because %s is more recent than %s."
% (newest_docket_date, a.date_sourced))
a.date_sourced = newest_docket_date
a.contact_raw = atty['contact']
a.email = atty_info['email']
a.phone = atty_info['phone']
a.fax = atty_info['fax']
a.save()
# Do roles
atty_roles = [normalize_attorney_role(r) for r in atty['roles']]
atty_roles = filter(lambda r: r['role'] is not None, atty_roles)
atty_roles = remove_duplicate_dicts(atty_roles)
if len(atty_roles) > 0:
logger.info("Linking attorney '%s' to party '%s' via %s roles: %s" %
(atty['name'], p.name, len(atty_roles), atty_roles))
else:
logger.info("No role data parsed. Linking via 'UNKNOWN' role.")
atty_roles = [{'role': Role.UNKNOWN, 'date_action': None}]
# Delete the old roles, replace with new.
Role.objects.filter(attorney=a, party=p, docket=d).delete()
Role.objects.bulk_create([
Role(attorney=a, party=p, docket=d, **atty_role) for
atty_role in atty_roles
])
return a
def test_attorney_role_normalization(self):
"""Can we normalize the attorney roles into a small number of roles?"""
pairs = [
{
"q": "(Inactive)",
"a": {
"role": Role.INACTIVE,
"date_action": None,
"role_raw": "(Inactive)",
},
},
{
"q": "ATTORNEY IN SEALED GROUP",
"a": {
"role": Role.ATTORNEY_IN_SEALED_GROUP,
"date_action": None,
"role_raw": "ATTORNEY IN SEALED GROUP",
},
},
{
"q": "ATTORNEY TO BE NOTICED",
"a": {
"role": Role.ATTORNEY_TO_BE_NOTICED,
"date_action": None,
"role_raw": "ATTORNEY TO BE NOTICED",
},
},
{
"q": "Bar Status: ACTIVE",
"a": {
"role": None,
"date_action": None,
"role_raw": "Bar Status: ACTIVE",
},
},
{
"q": "DISBARRED 02/19/2010",
"a": {
"role": Role.DISBARRED,
"date_action": datetime.date(2010, 2, 19),
"role_raw": "DISBARRED 02/19/2010",
},
},
{
"q": "Designation: ADR Pro Bono Limited Scope Counsel",
"a": {
"role": None,
"date_action": None,
"role_raw": "Designation: ADR Pro Bono Limited Scope "
"Counsel",
},
},
{
"q": "LEAD ATTORNEY",
"a": {
"role": Role.ATTORNEY_LEAD,
"date_action": None,
"role_raw": "LEAD ATTORNEY",
},
},
{
"q": "PRO HAC VICE",
"a": {
"role": Role.PRO_HAC_VICE,
"date_action": None,
"role_raw": "PRO HAC VICE",
},
},
{
"q": "SELF- TERMINATED: 01/14/2013",
"a": {
"role": Role.SELF_TERMINATED,
"date_action": datetime.date(2013, 1, 14),
"role_raw": "SELF- TERMINATED: 01/14/2013",
},
},
{
"q": "SUSPENDED 01/22/2016",
"a": {
"role": Role.SUSPENDED,
"date_action": datetime.date(2016, 1, 22),
"role_raw": "SUSPENDED 01/22/2016",
},
},
{
"q": "TERMINATED: 01/01/2007",
"a": {
"role": Role.TERMINATED,
"date_action": datetime.date(2007, 1, 1),
"role_raw": "TERMINATED: 01/01/2007",
},
},
{
"q": "Blagger jabber",
"a": {
"role": None,
"date_action": None,
"role_raw": "Blagger jabber",
},
},
]
for pair in pairs:
print(
"Normalizing PACER role of '%s' to '%s'..."
% (pair["q"], pair["a"]),
end="",
)
result = normalize_attorney_role(pair["q"])
self.assertEqual(result, pair["a"])
print("✓")
def add_attorney(atty, p, d):
"""Add/update an attorney.
Given an attorney node, and a party and a docket object, add the attorney
to the database or link the attorney to the new docket. Also add/update the
attorney organization, and the attorney's role in the case.
:param atty: A dict representing an attorney, as provided by Juriscraper.
:param p: A Party object
:param d: A Docket object
:return: None if there's an error, or an Attorney object if not.
"""
newest_docket_date = max([dt for dt in [d.date_filed, d.date_terminated,
d.date_last_filing] if dt])
atty_org_info, atty_info = normalize_attorney_contact(
atty['contact'],
fallback_name=atty['name'],
)
try:
q = Q()
fields = {
('phone', atty_info['phone']),
('fax', atty_info['fax']),
('email', atty_info['email']),
('contact_raw', atty['contact']),
('organizations__lookup_key', atty_org_info.get('lookup_key')),
}
for field, lookup in fields:
if lookup:
q |= Q(**{field: lookup})
a, created = Attorney.objects.filter(
Q(name=atty['name']) & q,
).get_or_create(
defaults={
'name': atty['name'],
'date_sourced': newest_docket_date,
'contact_raw': atty['contact'],
},
)
except Attorney.MultipleObjectsReturned:
logger.info("Got too many results for attorney: '%s'. Punting." % atty)
return None
# Associate the attorney with an org and update their contact info.
if atty['contact']:
if atty_org_info:
logger.info("Adding organization information to '%s': '%s'" %
(atty['name'], atty_org_info))
try:
org = AttorneyOrganization.objects.get(
lookup_key=atty_org_info['lookup_key'],
)
except AttorneyOrganization.DoesNotExist:
org = AttorneyOrganization.objects.create(**atty_org_info)
# Add the attorney to the organization
AttorneyOrganizationAssociation.objects.get_or_create(
attorney=a,
attorney_organization=org,
docket=d,
)
docket_info_is_newer = (a.date_sourced <= newest_docket_date)
if atty_info and docket_info_is_newer:
logger.info("Updating atty info because %s is more recent than %s."
% (newest_docket_date, a.date_sourced))
a.date_sourced = newest_docket_date
a.contact_raw = atty['contact']
a.email = atty_info['email']
a.phone = atty_info['phone']
a.fax = atty_info['fax']
a.save()
# Do roles
atty_roles = [normalize_attorney_role(r) for r in atty['roles']]
atty_roles = filter(lambda r: r['role'] is not None, atty_roles)
atty_roles = remove_duplicate_dicts(atty_roles)
if len(atty_roles) > 0:
logger.info("Linking attorney '%s' to party '%s' via %s roles: %s" %
(atty['name'], p.name, len(atty_roles), atty_roles))
else:
logger.info("No role data parsed. Linking via 'UNKNOWN' role.")
atty_roles = [{'role': Role.UNKNOWN, 'date_action': None}]
# Delete the old roles, replace with new.
Role.objects.filter(attorney=a, party=p, docket=d).delete()
Role.objects.bulk_create([
Role(attorney=a, party=p, docket=d, **atty_role) for
atty_role in atty_roles
])
return a
