def match_person(
*,
ingested_person: entities.Person,
db_people: List[entities.Person],
orphaned_entities: List[Entity],
matched_people_by_db_id: Dict[int, entities.Person]) -> None:
"""
Finds the best match for the provided |ingested_person| from the
provided |db_people|. If a match exists, the primary key is added onto
the |ingested_person| and then we attempt to match all children
entities.
"""
db_person = cast(entities.Person,
get_best_match(ingested_person,
db_people,
county_matching_utils.is_person_match,
matched_people_by_db_id.keys()))
if db_person:
person_id = db_person.get_id()
logging.debug('Successfully matched to person with ID %s',
person_id)
# If the match was previously matched to a different database
# person, raise an error.
if person_id in matched_people_by_db_id:
matches = [ingested_person,
matched_people_by_db_id[person_id]]
raise MatchedMultipleIngestedEntitiesError(db_person, matches)
ingested_person.set_id(person_id)
matched_people_by_db_id[cast(int, person_id)] = ingested_person
match_bookings(db_person=db_person,
ingested_person=ingested_person,
orphaned_entities=orphaned_entities)
def remove_pii_for_person(person: county_entities.Person) -> None:
"""Removes all of the PII for a person
Args:
person: (entities.Person) The entity object to scrub.
"""
person.full_name = None
if person.birthdate:
person.birthdate = datetime.date(
year=person.birthdate.year,
month=1,
day=1,
)
def testConvert_PersonInferredBooking(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add()
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
bookings=[
Booking.new_with_defaults(
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
admission_date=_INGEST_TIME.date(),
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
)
]
)
]
self.assertEqual(result, expected_result)
def testConvert_ExternalId_ClearPII(self):
# Arrange
metadata = IngestMetadata('REGION', _JURISDICTION_ID, _INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(person_id='PERSON_ID',
full_name='full_name',
booking_ids=['BOOKING_ID'])
ingest_info.bookings.add(booking_id='BOOKING_ID',
admission_date=str(_RELEASE_DATE))
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
external_id='PERSON_ID',
region='REGION',
jurisdiction_id='JURISDICTION_ID',
bookings=[
Booking.new_with_defaults(
external_id='BOOKING_ID',
admission_date=_RELEASE_DATE,
admission_date_inferred=False,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME)
])
]
self.assertEqual(result, expected_result)
def test_class_id_name_county_entity_sample(self):
self.assertEqual('person_id',
Person.get_class_id_name())
self.assertEqual('booking_id',
Booking.get_class_id_name())
self.assertEqual('hold_id',
Hold.get_class_id_name())
def testConvert_TotalBondNoCharge_CreatesChargeWithTotalBondAmount(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=['BOOKING_ID'])
ingest_info.bookings.add(booking_id='BOOKING_ID',
total_bond_amount='$100')
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(bookings=[
Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
external_id='BOOKING_ID',
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
charges=[
Charge.new_with_defaults(
status=ChargeStatus.PRESENT_WITHOUT_INFO,
bond=Bond.new_with_defaults(
status=BondStatus.PRESENT_WITHOUT_INFO,
bond_type=BondType.CASH,
amount_dollars=100))
])
])
]
self.assertEqual(result, expected_result)
def test_get_entity_name_county_entity_sample(self):
self.assertEqual('person',
Person.new_with_defaults().get_entity_name())
self.assertEqual('booking',
Booking.new_with_defaults().get_entity_name())
self.assertEqual('hold',
Hold.new_with_defaults().get_entity_name())
def testConvert_FullIngestInfo_GeneratedIds(self):
# Arrange
metadata = IngestMetadata('REGION', _JURISDICTION_ID, _INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(person_id='PERSON_ID_GENERATE',
booking_ids=['BOOKING_ID_GENERATE'])
ingest_info.bookings.add(
booking_id='BOOKING_ID_GENERATE',
arrest_id='ARREST_ID_GENERATE',
hold_ids=['HOLD_ID_1_GENERATE', 'HOLD_ID_2_GENERATE'],
charge_ids=['CHARGE_ID_GENERATE'])
ingest_info.holds.add(hold_id='HOLD_ID_1_GENERATE',
jurisdiction_name='jurisdiction')
ingest_info.holds.add(hold_id='HOLD_ID_2_GENERATE',
jurisdiction_name='jurisdiction')
ingest_info.arrests.add(arrest_id='ARREST_ID_GENERATE', agency='PD')
ingest_info.charges.add(charge_id='CHARGE_ID_GENERATE',
name='DUI',
bond_id='BOND_ID_GENERATE',
sentence_id='SENTENCE_ID_GENERATE')
ingest_info.bonds.add(bond_id='BOND_ID_GENERATE')
ingest_info.sentences.add(sentence_id='SENTENCE_ID_GENERATE',
is_life='True')
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
region='REGION',
jurisdiction_id='JURISDICTION_ID',
bookings=[
Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
arrest=Arrest.new_with_defaults(agency='PD'),
holds=[
Hold.new_with_defaults(
jurisdiction_name='JURISDICTION',
status=HoldStatus.PRESENT_WITHOUT_INFO)
],
charges=[
Charge.new_with_defaults(
status=ChargeStatus.PRESENT_WITHOUT_INFO,
name='DUI',
bond=Bond.new_with_defaults(
status=BondStatus.PRESENT_WITHOUT_INFO),
sentence=Sentence.new_with_defaults(
status=SentenceStatus.PRESENT_WITHOUT_INFO,
is_life=True))
])
])
]
self.assertEqual(result, expected_result)
def test_get_id_county_entity_sample(self):
self.assertEqual(123,
Person.new_with_defaults(person_id=123).get_id())
self.assertEqual(456,
Booking.new_with_defaults(booking_id=456).get_id())
self.assertEqual(789,
Hold.new_with_defaults(hold_id=789).get_id())
self.assertIsNone(Charge.new_with_defaults().get_id())
def testConvert_TotalBondWithCharge_SetsTotalBondOnCharge(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=["BOOKING_ID"])
ingest_info.bookings.add(booking_id="BOOKING_ID",
total_bond_amount="$100",
charge_ids=["CHARGE_ID"])
ingest_info.charges.add(charge_id="CHARGE_ID")
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
bookings=[
Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
charges=[
Charge.new_with_defaults(
external_id="CHARGE_ID_COUNT_1",
status=ChargeStatus.PRESENT_WITHOUT_INFO,
bond=Bond.new_with_defaults(
amount_dollars=100,
status=BondStatus.PRESENT_WITHOUT_INFO,
bond_type=BondType.CASH,
),
)
],
)
],
)
]
self.assertEqual(result, expected_result)
def test_class_id_name_county_entity_sample(self) -> None:
self.assertEqual("person_id", Person.get_class_id_name())
self.assertEqual("booking_id", Booking.get_class_id_name())
self.assertEqual("hold_id", Hold.get_class_id_name())
def testConvert_MultipleCountsOfCharge_CreatesDuplicateCharges(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=['BOOKING_ID'])
ingest_info.bookings.add(booking_id='BOOKING_ID',
charge_ids=['CHARGE_ID'])
ingest_info.charges.add(charge_id='CHARGE_ID',
name='CHARGE_NAME',
number_of_counts='3',
bond_id='BOND_ID')
ingest_info.bonds.add(bond_id='BOND_ID')
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_duplicate_charge = Charge.new_with_defaults(
external_id='CHARGE_ID_COUNT_1',
status=ChargeStatus.PRESENT_WITHOUT_INFO,
name='CHARGE_NAME')
expected_bond = Bond.new_with_defaults(
external_id='BOND_ID', status=BondStatus.PRESENT_WITHOUT_INFO)
expected_result = [
Person.new_with_defaults(bookings=[
Booking.new_with_defaults(
external_id='BOOKING_ID',
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
admission_date_inferred=True,
admission_date=_INGEST_TIME.date(),
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
charges=[
attr.evolve(expected_duplicate_charge,
external_id='CHARGE_ID_COUNT_1',
bond=expected_bond),
attr.evolve(expected_duplicate_charge,
external_id='CHARGE_ID_COUNT_2',
bond=expected_bond),
attr.evolve(expected_duplicate_charge,
external_id='CHARGE_ID_COUNT_3',
bond=expected_bond),
])
])
]
self.assertEqual(result, expected_result)
# Assert that the expanded charges, while containing duplicate
# information, are actually different objects.
# For some reason, pylint is having trouble identifying that you can
# index into the result list even though the revealed type is a list.
# pylint: disable=unsubscriptable-object
result_expanded_charges = result[0].bookings[0].charges
charges_grouped_by_id = list(
itertools.groupby(result_expanded_charges, key=id))
self.assertEqual(len(result_expanded_charges),
len(charges_grouped_by_id))
# Assert that the bond belonging to each charge is the same object.
bonds = [charge.bond for charge in result_expanded_charges]
self.assertTrue(all(bond is bonds[0] for bond in bonds))
def testConvert_FullIngestInfo_NoOpenBookings(self):
# Arrange
metadata = IngestMetadata('REGION', _JURISDICTION_ID, _INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(person_id='PERSON_ID',
full_name='TEST',
birthdate=str(_BIRTHDATE),
booking_ids=['BOOKING_ID'])
ingest_info.bookings.add(booking_id='BOOKING_ID',
arrest_id='ARREST_ID',
release_date=str(_RELEASE_DATE),
charge_ids=['CHARGE_ID'])
ingest_info.arrests.add(arrest_id='ARREST_ID', agency='PD')
ingest_info.charges.add(charge_id='CHARGE_ID',
name='DUI',
bond_id='BOND_ID',
sentence_id='SENTENCE_ID')
ingest_info.bonds.add(bond_id='BOND_ID')
ingest_info.sentences.add(sentence_id='SENTENCE_ID', is_life='True')
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
external_id='PERSON_ID',
region='REGION',
jurisdiction_id='JURISDICTION_ID',
birthdate=_BIRTHDATE_SCRUBBED,
birthdate_inferred_from_age=False,
bookings=[
Booking.new_with_defaults(
external_id='BOOKING_ID',
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
release_date=_RELEASE_DATE,
release_date_inferred=False,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.RELEASED,
arrest=Arrest.new_with_defaults(
external_id='ARREST_ID', agency='PD'),
charges=[
Charge.new_with_defaults(
external_id='CHARGE_ID_COUNT_1',
status=ChargeStatus.PRESENT_WITHOUT_INFO,
name='DUI',
bond=Bond.new_with_defaults(
external_id='BOND_ID',
status=BondStatus.PRESENT_WITHOUT_INFO),
sentence=Sentence.new_with_defaults(
status=SentenceStatus.PRESENT_WITHOUT_INFO,
external_id='SENTENCE_ID',
is_life=True))
])
])
]
self.assertEqual(result, expected_result)
def testConvert_FullIngestInfo_NoOpenBookings(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id=_JURISDICTION_ID,
ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(
person_id="PERSON_ID",
full_name="TEST",
birthdate=str(_BIRTHDATE),
booking_ids=["BOOKING_ID"],
)
ingest_info.bookings.add(
booking_id="BOOKING_ID",
arrest_id="ARREST_ID",
release_date=str(_RELEASE_DATE),
charge_ids=["CHARGE_ID"],
)
ingest_info.arrests.add(arrest_id="ARREST_ID", agency="PD")
ingest_info.charges.add(
charge_id="CHARGE_ID",
name="DUI",
bond_id="BOND_ID",
sentence_id="SENTENCE_ID",
)
ingest_info.bonds.add(bond_id="BOND_ID")
ingest_info.sentences.add(sentence_id="SENTENCE_ID", is_life="True")
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
external_id="PERSON_ID",
region="REGION",
jurisdiction_id="JURISDICTION_ID",
birthdate=_BIRTHDATE_SCRUBBED,
birthdate_inferred_from_age=False,
bookings=[
Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
release_date=_RELEASE_DATE,
release_date_inferred=False,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.RELEASED,
arrest=Arrest.new_with_defaults(
external_id="ARREST_ID", agency="PD"),
charges=[
Charge.new_with_defaults(
external_id="CHARGE_ID_COUNT_1",
status=ChargeStatus.PRESENT_WITHOUT_INFO,
name="DUI",
bond=Bond.new_with_defaults(
external_id="BOND_ID",
status=BondStatus.PRESENT_WITHOUT_INFO,
),
sentence=Sentence.new_with_defaults(
status=SentenceStatus.PRESENT_WITHOUT_INFO,
external_id="SENTENCE_ID",
is_life=True,
),
)
],
)
],
)
]
self.assertEqual(result, expected_result)
def testConvert_FullIngestInfo(self):
# Arrange
metadata = IngestMetadata(
"REGION", _JURISDICTION_ID, _INGEST_TIME, facility_id=_FACILITY_ID
)
ingest_info = IngestInfo()
ingest_info.people.add(person_id="PERSON_ID", booking_ids=["BOOKING_ID"])
ingest_info.bookings.add(
booking_id="BOOKING_ID", arrest_id="ARREST_ID", charge_ids=["CHARGE_ID"]
)
ingest_info.arrests.add(arrest_id="ARREST_ID", agency="PD")
ingest_info.charges.add(
charge_id="CHARGE_ID",
name="DUI",
bond_id="BOND_ID",
sentence_id="SENTENCE_ID",
)
ingest_info.bonds.add(bond_id="BOND_ID")
ingest_info.sentences.add(sentence_id="SENTENCE_ID", is_life="True")
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
external_id="PERSON_ID",
region="REGION",
jurisdiction_id="JURISDICTION_ID",
bookings=[
Booking.new_with_defaults(
external_id="BOOKING_ID",
facility_id=_FACILITY_ID,
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
arrest=Arrest.new_with_defaults(
external_id="ARREST_ID", agency="PD"
),
charges=[
Charge.new_with_defaults(
external_id="CHARGE_ID_COUNT_1",
status=ChargeStatus.PRESENT_WITHOUT_INFO,
name="DUI",
bond=Bond.new_with_defaults(
external_id="BOND_ID",
status=BondStatus.PRESENT_WITHOUT_INFO,
),
sentence=Sentence.new_with_defaults(
status=SentenceStatus.PRESENT_WITHOUT_INFO,
external_id="SENTENCE_ID",
is_life=True,
),
)
],
)
],
)
]
self.assertEqual(result, expected_result)
def test_get_entity_name_county_entity_sample(self) -> None:
self.assertEqual("person",
Person.new_with_defaults().get_entity_name())
self.assertEqual("booking",
Booking.new_with_defaults().get_entity_name())
self.assertEqual("hold", Hold.new_with_defaults().get_entity_name())
def testConvert_FullIngestInfo_GeneratedIds(self):
# Arrange
metadata = IngestMetadata("REGION", _JURISDICTION_ID, _INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(
person_id="PERSON_ID_GENERATE", booking_ids=["BOOKING_ID_GENERATE"]
)
ingest_info.bookings.add(
booking_id="BOOKING_ID_GENERATE",
arrest_id="ARREST_ID_GENERATE",
hold_ids=["HOLD_ID_1_GENERATE", "HOLD_ID_2_GENERATE"],
charge_ids=["CHARGE_ID_GENERATE"],
)
ingest_info.holds.add(
hold_id="HOLD_ID_1_GENERATE", jurisdiction_name="jurisdiction"
)
ingest_info.holds.add(
hold_id="HOLD_ID_2_GENERATE", jurisdiction_name="jurisdiction"
)
ingest_info.arrests.add(arrest_id="ARREST_ID_GENERATE", agency="PD")
ingest_info.charges.add(
charge_id="CHARGE_ID_GENERATE",
name="DUI",
bond_id="BOND_ID_GENERATE",
sentence_id="SENTENCE_ID_GENERATE",
)
ingest_info.bonds.add(bond_id="BOND_ID_GENERATE")
ingest_info.sentences.add(sentence_id="SENTENCE_ID_GENERATE", is_life="True")
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
bookings=[
Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
arrest=Arrest.new_with_defaults(agency="PD"),
holds=[
Hold.new_with_defaults(
jurisdiction_name="JURISDICTION",
status=HoldStatus.PRESENT_WITHOUT_INFO,
)
],
charges=[
Charge.new_with_defaults(
status=ChargeStatus.PRESENT_WITHOUT_INFO,
name="DUI",
bond=Bond.new_with_defaults(
status=BondStatus.PRESENT_WITHOUT_INFO
),
sentence=Sentence.new_with_defaults(
status=SentenceStatus.PRESENT_WITHOUT_INFO,
is_life=True,
),
)
],
)
],
)
]
self.assertEqual(result, expected_result)