def match_and_write_people(session: Session) -> bool:
logging.info("Starting entity matching")
entity_matching_output = entity_matching.match(
session, metadata.region, people)
output_people = entity_matching_output.people
total_root_entities = total_people \
if metadata.system_level == SystemLevel.COUNTY \
else entity_matching_output.total_root_entities
logging.info("Completed entity matching with [%s] errors",
entity_matching_output.error_count)
logging.info(
"Completed entity matching and have [%s] total people "
"to commit to DB", len(output_people))
if _should_abort(
total_root_entities=total_root_entities,
conversion_result=conversion_result,
entity_matching_errors=entity_matching_output.error_count,
data_validation_errors=data_validation_errors):
# TODO(#1665): remove once dangling PERSIST session
# investigation is complete.
logging.info("_should_abort_ was true after entity matching")
return False
database.write_people(
session,
output_people,
metadata,
orphaned_entities=entity_matching_output.orphaned_entities)
logging.info("Successfully wrote to the database")
return True
def test_matchPerson_updateStatusOnOrphanedEntities(self):
# Arrange
schema_bond = schema.Bond(
bond_id=_BOND_ID, status=BondStatus.PENDING.value,
booking_id=_BOOKING_ID)
schema_charge = schema.Charge(
charge_id=_CHARGE_ID, status=ChargeStatus.PENDING.value,
bond=schema_bond)
schema_booking = schema.Booking(
admission_date=_DATE_2, booking_id=_BOOKING_ID,
custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE,
first_seen_time=_DATE, charges=[schema_charge])
schema_person = schema.Person(
person_id=_PERSON_ID, full_name=_FULL_NAME, birthdate=_DATE,
jurisdiction_id=_JURISDICTION_ID, region=_REGION,
bookings=[schema_booking])
session = SessionFactory.for_schema_base(JailsBase)
session.add(schema_person)
session.commit()
ingested_charge_no_bond = attr.evolve(
converter.convert_schema_object_to_entity(schema_charge),
charge_id=None,
bond=None)
ingested_booking = attr.evolve(
converter.convert_schema_object_to_entity(schema_booking),
booking_id=None,
custody_status=CustodyStatus.RELEASED,
charges=[ingested_charge_no_bond])
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person),
person_id=None,
bookings=[ingested_booking])
# Act
out = entity_matching.match(session, _REGION, [ingested_person])
# Assert
expected_orphaned_bond = attr.evolve(
converter.convert_schema_object_to_entity(schema_bond),
status=BondStatus.REMOVED_WITHOUT_INFO)
expected_charge = attr.evolve(
ingested_charge_no_bond, charge_id=schema_charge.charge_id)
expected_booking = attr.evolve(
ingested_booking, booking_id=schema_booking.booking_id,
charges=[expected_charge])
expected_person = attr.evolve(
ingested_person, person_id=schema_person.person_id,
bookings=[expected_booking])
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.people),
[expected_person])
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.orphaned_entities),
[expected_orphaned_bond])
self.assertEqual(out.error_count, 0)
def test_matchPeople_errorCount(self):
# Arrange
schema_booking = schema.Booking(
external_id=_EXTERNAL_ID, admission_date=_DATE_2,
booking_id=_BOOKING_ID,
custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE,
first_seen_time=_DATE)
schema_booking_another = copy.deepcopy(schema_booking)
schema_booking_another.booking_id = _BOOKING_ID_ANOTHER
schema_person = schema.Person(
person_id=_PERSON_ID, external_id=_EXTERNAL_ID,
jurisdiction_id=_JURISDICTION_ID,
full_name=_FULL_NAME, birthdate=_DATE,
region=_REGION, bookings=[schema_booking, schema_booking_another])
schema_person_another = schema.Person(person_id=_PERSON_ID_ANOTHER,
jurisdiction_id=_JURISDICTION_ID,
region=_REGION,
full_name=_NAME_2,
external_id=_EXTERNAL_ID_ANOTHER)
session = SessionFactory.for_schema_base(JailsBase)
session.add(schema_person)
session.add(schema_person_another)
session.commit()
ingested_booking = attr.evolve(
converter.convert_schema_object_to_entity(schema_booking),
booking_id=None,
custody_status=CustodyStatus.RELEASED)
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person),
person_id=None,
bookings=[ingested_booking])
ingested_person_another = attr.evolve(
converter.convert_schema_object_to_entity(schema_person_another),
person_id=None
)
# Act
out = entity_matching.match(
session, _REGION, [ingested_person, ingested_person_another])
# Assert
expected_person = attr.evolve(ingested_person_another,
person_id=schema_person_another.person_id)
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.people),
[expected_person])
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.orphaned_entities),
[])
self.assertEqual(out.error_count, 1)
def test_matchPeople_twoMatchingPeople_PicksMostSimilar(self):
# Arrange
schema_person = schema.Person(
person_id=_PERSON_ID,
external_id=_EXTERNAL_ID,
jurisdiction_id=_JURISDICTION_ID,
full_name=_FULL_NAME,
birthdate=_DATE,
region=_REGION,
gender=Gender.MALE.value,
)
schema_person_mismatch = copy.deepcopy(schema_person)
schema_person_mismatch.person_id = _PERSON_ID_ANOTHER
schema_person_mismatch.gender = Gender.FEMALE.value
session = SessionFactory.for_schema_base(JailsBase)
session.add(schema_person)
session.add(schema_person_mismatch)
session.commit()
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person), person_id=None
)
expected_person = attr.evolve(
ingested_person, person_id=schema_person.person_id
)
# Act
matched_entities = entity_matching.match(session, _REGION, [ingested_person])
# Assert both schema objects are matches, but we select the most
# similar one.
self.assertTrue(
county_matching_utils.is_person_match(
db_entity=schema_person, ingested_entity=ingested_person
)
)
self.assertTrue(
county_matching_utils.is_person_match(
db_entity=schema_person_mismatch, ingested_entity=ingested_person
)
)
self.assertEqual(matched_entities.error_count, 0)
self.assertEqual(len(matched_entities.orphaned_entities), 0)
self.assertEqual(ingested_person, expected_person)
def test_matchPeople(self):
# Arrange
schema_booking = schema.Booking(
admission_date=_DATE_2,
booking_id=_BOOKING_ID,
custody_status=CustodyStatus.IN_CUSTODY.value,
last_seen_time=_DATE,
first_seen_time=_DATE,
)
schema_person = schema.Person(
person_id=_PERSON_ID,
full_name=_FULL_NAME,
birthdate=_DATE,
jurisdiction_id=_JURISDICTION_ID,
region=_REGION,
bookings=[schema_booking],
)
schema_booking_external_id = schema.Booking(
admission_date=_DATE_2,
booking_id=_BOOKING_ID_ANOTHER,
release_date=_DATE,
custody_status=CustodyStatus.RELEASED.value,
last_seen_time=_DATE,
first_seen_time=_DATE,
)
schema_person_external_id = schema.Person(
person_id=_PERSON_ID_ANOTHER,
external_id=_EXTERNAL_ID,
full_name=_FULL_NAME,
birthdate=_DATE,
jurisdiction_id=_JURISDICTION_ID,
region=_REGION,
bookings=[schema_booking_external_id],
)
with SessionFactory.using_database(self.database_key,
autocommit=False) as session:
session.add(schema_person)
session.add(schema_person_external_id)
session.commit()
ingested_booking = attr.evolve(
converter.convert_schema_object_to_entity(schema_booking),
booking_id=None,
custody_status=CustodyStatus.RELEASED,
)
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person),
person_id=None,
bookings=[ingested_booking],
)
ingested_booking_external_id = attr.evolve(
converter.convert_schema_object_to_entity(
schema_booking_external_id),
booking_id=None,
facility=_FACILITY,
)
ingested_person_external_id = attr.evolve(
converter.convert_schema_object_to_entity(
schema_person_external_id),
person_id=None,
bookings=[ingested_booking_external_id],
)
# Act
out = entity_matching.match(
session, _REGION, [ingested_person_external_id, ingested_person])
# Assert
expected_booking = attr.evolve(ingested_booking,
booking_id=_BOOKING_ID)
expected_person = attr.evolve(ingested_person,
person_id=_PERSON_ID,
bookings=[expected_booking])
expected_booking_external_id = attr.evolve(
ingested_booking_external_id, booking_id=_BOOKING_ID_ANOTHER)
expected_person_external_id = attr.evolve(
ingested_person_external_id,
person_id=_PERSON_ID_ANOTHER,
bookings=[expected_booking_external_id],
)
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.people),
[expected_person_external_id, expected_person],
)
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.orphaned_entities),
[])
self.assertEqual(out.error_count, 0)
def write(ingest_info, metadata):
"""
If in prod or if 'PERSIST_LOCALLY' is set to true, persist each person in
the ingest_info. If a person with the given surname/birthday already exists,
then update that person.
Otherwise, simply log the given ingest_infos for debugging
"""
ingest_info_validator.validate(ingest_info)
mtags = {
monitoring.TagKey.SHOULD_PERSIST: _should_persist(),
monitoring.TagKey.PERSISTED: False
}
total_people = _get_total_people(ingest_info, metadata)
with monitoring.measurements(mtags) as measurements:
# Convert the people one at a time and count the errors as they happen.
conversion_result: IngestInfoConversionResult = \
ingest_info_converter.convert_to_persistence_entities(ingest_info,
metadata)
people, data_validation_errors = entity_validator.validate(
conversion_result.people)
logging.info(
"Converted [%s] people with [%s] enum_parsing_errors, [%s]"
" general_parsing_errors, [%s] protected_class_errors and "
"[%s] data_validation_errors", len(people),
conversion_result.enum_parsing_errors,
conversion_result.general_parsing_errors,
conversion_result.protected_class_errors, data_validation_errors)
measurements.measure_int_put(m_people, len(people))
if _should_abort(total_root_entities=total_people,
conversion_result=conversion_result,
data_validation_errors=data_validation_errors):
# TODO(#1665): remove once dangling PERSIST session investigation
# is complete.
logging.info("_should_abort_ was true after converting people")
return False
if not _should_persist():
return True
persisted = False
session = SessionFactory.for_schema_base(
schema_base_for_system_level(metadata.system_level))
try:
logging.info("Starting entity matching")
entity_matching_output = entity_matching.match(
session, metadata.region, people)
people = entity_matching_output.people
total_root_entities = total_people \
if metadata.system_level == SystemLevel.COUNTY \
else entity_matching_output.total_root_entities
logging.info("Completed entity matching with [%s] errors",
entity_matching_output.error_count)
logging.info(
"Completed entity matching and have [%s] total people "
"to commit to DB", len(people))
if _should_abort(
total_root_entities=total_root_entities,
conversion_result=conversion_result,
entity_matching_errors=entity_matching_output.error_count,
data_validation_errors=data_validation_errors):
# TODO(#1665): remove once dangling PERSIST session
# investigation is complete.
logging.info("_should_abort_ was true after entity matching")
return False
database.write_people(
session,
people,
metadata,
orphaned_entities=entity_matching_output.orphaned_entities)
logging.info("Successfully wrote to the database")
session.commit()
persisted = True
mtags[monitoring.TagKey.PERSISTED] = True
except Exception as e:
logging.exception("An exception was raised in write(): [%s]",
type(e).__name__)
# Record the error type that happened and increment the counter
mtags[monitoring.TagKey.ERROR] = type(e).__name__
measurements.measure_int_put(m_errors, 1)
session.rollback()
raise
finally:
session.close()
return persisted
def test_supervisionViolationsWithDifferentParents_mergesViolations(self):
db_person = generate_person(person_id=_ID, full_name=_FULL_NAME)
db_supervision_violation = generate_supervision_violation(
person=db_person,
state_code=_US_MO,
supervision_violation_id=_ID,
external_id=_EXTERNAL_ID)
db_placeholder_supervision_period = generate_supervision_period(
person=db_person,
state_code=_US_MO,
supervision_period_id=_ID,
supervision_violation_entries=[db_supervision_violation])
db_incarceration_sentence = generate_incarceration_sentence(
person=db_person,
state_code=_US_MO,
incarceration_sentence_id=_ID,
external_id=_EXTERNAL_ID,
supervision_periods=[db_placeholder_supervision_period])
db_sentence_group = generate_sentence_group(
person=db_person,
state_code=_US_MO,
sentence_group_id=_ID,
external_id=_EXTERNAL_ID,
incarceration_sentences=[db_incarceration_sentence])
db_external_id = generate_external_id(person=db_person,
person_external_id_id=_ID,
state_code=_US_MO,
external_id=_EXTERNAL_ID)
db_person.sentence_groups = [db_sentence_group]
db_person.external_ids = [db_external_id]
self._commit_to_db(db_person)
supervision_violation = attr.evolve(
self.to_entity(db_supervision_violation),
supervision_violation_id=None,
external_id=_EXTERNAL_ID_WITH_SUFFIX)
placeholder_supervision_period = attr.evolve(
self.to_entity(db_placeholder_supervision_period),
supervision_period_id=None,
supervision_violation_entries=[supervision_violation])
supervision_sentence = StateSupervisionSentence.new_with_defaults(
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
supervision_periods=[placeholder_supervision_period])
sentence_group = attr.evolve(
self.to_entity(db_sentence_group),
sentence_group_id=None,
incarceration_sentences=[],
supervision_sentences=[supervision_sentence])
external_id = attr.evolve(self.to_entity(db_external_id),
person_external_id_id=None)
person = attr.evolve(self.to_entity(db_person),
person_id=None,
sentence_groups=[sentence_group],
external_ids=[external_id])
expected_supervision_violation = attr.evolve(
self.to_entity(db_supervision_violation))
expected_placeholder_supervision_period_is = attr.evolve(
placeholder_supervision_period,
supervision_violation_entries=[expected_supervision_violation])
expected_placeholder_supervision_period_ss = attr.evolve(
self.to_entity(db_placeholder_supervision_period),
supervision_violation_entries=[expected_supervision_violation])
expected_supervision_sentence = attr.evolve(
supervision_sentence,
supervision_periods=[expected_placeholder_supervision_period_ss])
expected_incarceration_sentence = attr.evolve(
self.to_entity(db_incarceration_sentence),
supervision_periods=[expected_placeholder_supervision_period_is])
expected_sentence_group = attr.evolve(
self.to_entity(db_sentence_group),
incarceration_sentences=[expected_incarceration_sentence],
supervision_sentences=[expected_supervision_sentence])
expected_external_id = attr.evolve(self.to_entity(db_external_id))
expected_person = attr.evolve(
self.to_entity(db_person),
external_ids=[expected_external_id],
sentence_groups=[expected_sentence_group])
# Act 1 - Match
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session)
self.assertEqual(0, matched_entities.error_count)
self.assertEqual(1, matched_entities.total_root_entities)
def test_runMatch_supervisionPeriodDateChangesSoItDoesNotMatchSentenceOrViolations(
self):
# Arrange
db_supervising_officer = generate_agent(
agent_id=_ID,
external_id=_EXTERNAL_ID,
state_code=_US_MO,
agent_type=StateAgentType.SUPERVISION_OFFICER.value)
db_person = generate_person(person_id=_ID,
supervising_officer=db_supervising_officer)
db_external_id = generate_external_id(person_external_id_id=_ID,
external_id=_EXTERNAL_ID,
state_code=_US_MO,
id_type=_ID_TYPE)
# Violation has been date matched to the open supervision period
db_supervision_violation = generate_supervision_violation(
person=db_person,
state_code=_US_MO,
supervision_violation_id=_ID,
external_id=_EXTERNAL_ID,
violation_date=_DATE_4)
db_supervision_period_open = generate_supervision_period(
person=db_person,
supervision_period_id=_ID_2,
external_id=_EXTERNAL_ID_2,
start_date=_DATE_2,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
supervising_officer=db_supervising_officer,
supervision_violation_entries=[db_supervision_violation])
db_supervision_sentence = generate_supervision_sentence(
person=db_person,
external_id=_EXTERNAL_ID,
supervision_sentence_id=_ID,
state_code=_US_MO,
start_date=_DATE_1,
supervision_periods=[db_supervision_period_open])
db_sentence_group = generate_sentence_group(
external_id=_EXTERNAL_ID,
sentence_group_id=_ID,
state_code=_US_MO,
supervision_sentences=[db_supervision_sentence])
db_person.external_ids = [db_external_id]
db_person.sentence_groups = [db_sentence_group]
self._commit_to_db(db_person)
supervsion_period_updated = StateSupervisionPeriod.new_with_defaults(
state_code=_US_MO,
external_id=db_supervision_period_open.external_id,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO,
start_date=_DATE_2,
termination_date=_DATE_3,
)
placeholder_supervision_sentence = StateSupervisionSentence.new_with_defaults(
state_code=_US_MO, supervision_periods=[supervsion_period_updated])
sentence_group = StateSentenceGroup.new_with_defaults(
external_id=db_sentence_group.external_id,
state_code=_US_MO,
supervision_sentences=[placeholder_supervision_sentence])
external_id = StatePersonExternalId.new_with_defaults(
external_id=db_external_id.external_id,
state_code=_US_MO,
id_type=db_external_id.id_type)
person = StatePerson.new_with_defaults(
external_ids=[external_id], sentence_groups=[sentence_group])
expected_person = attr.evolve(self.to_entity(db_person))
expected_sentence = expected_person.sentence_groups[
0].supervision_sentences[0]
expected_original_supervision_period = expected_sentence.supervision_periods[
0]
# Violation is moved off of the supervision period (it no longer matches) and the termination date is updated
expected_original_supervision_period.supervision_violation_entries = []
expected_original_supervision_period.termination_date = _DATE_3
# A placeholder periods is created to hold the existing supervision violation
expected_new_placeholder_supervision_period = StateSupervisionPeriod.new_with_defaults(
state_code=_US_MO,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO,
supervision_violation_entries=[
self.to_entity(db_supervision_violation)
])
expected_sentence.supervision_periods.append(
expected_new_placeholder_supervision_period)
# Act
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
# Assert
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session,
debug=True)
self.assert_no_errors(matched_entities)
self.assertEqual(1, matched_entities.total_root_entities)
def test_runMatch_supervisingOfficerMovedFromSupervisionPeriodToPerson(
self):
# Arrange
db_supervising_officer = generate_agent(
agent_id=_ID,
external_id=_EXTERNAL_ID,
state_code=_US_MO,
agent_type=StateAgentType.SUPERVISION_OFFICER.value)
db_person = generate_person(person_id=_ID,
supervising_officer=db_supervising_officer)
db_external_id = generate_external_id(person_external_id_id=_ID,
external_id=_EXTERNAL_ID,
state_code=_US_MO,
id_type=_ID_TYPE)
db_supervision_period = generate_supervision_period(
person=db_person,
supervision_period_id=_ID,
external_id=_EXTERNAL_ID,
start_date=_DATE_1,
termination_date=_DATE_2,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
supervising_officer=db_supervising_officer)
db_supervision_period_open = generate_supervision_period(
person=db_person,
supervision_period_id=_ID_2,
external_id=_EXTERNAL_ID_2,
start_date=_DATE_2,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
supervising_officer=db_supervising_officer)
db_supervision_sentence = generate_supervision_sentence(
person=db_person,
external_id=_EXTERNAL_ID,
supervision_sentence_id=_ID,
state_code=_US_MO,
start_date=_DATE_1,
supervision_periods=[
db_supervision_period, db_supervision_period_open
])
db_sentence_group = generate_sentence_group(
external_id=_EXTERNAL_ID,
sentence_group_id=_ID,
state_code=_US_MO,
supervision_sentences=[db_supervision_sentence])
db_person.external_ids = [db_external_id]
db_person.sentence_groups = [db_sentence_group]
self._commit_to_db(db_person)
new_supervising_officer = StateAgent.new_with_defaults(
external_id=_EXTERNAL_ID_2,
state_code=_US_MO,
agent_type=StateAgentType.SUPERVISION_OFFICER)
new_supervision_period = StateSupervisionPeriod.new_with_defaults(
external_id=_EXTERNAL_ID_3,
state_code=_US_MO,
start_date=_DATE_3,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO,
supervising_officer=new_supervising_officer)
supervision_period_update = StateSupervisionPeriod.new_with_defaults(
external_id=db_supervision_period_open.external_id,
state_code=_US_MO,
termination_date=_DATE_3)
supervision_sentence = StateSupervisionSentence.new_with_defaults(
external_id=db_supervision_sentence.external_id,
state_code=_US_MO,
supervision_periods=[
supervision_period_update, new_supervision_period
])
sentence_group = StateSentenceGroup.new_with_defaults(
external_id=db_sentence_group.external_id,
state_code=_US_MO,
supervision_sentences=[supervision_sentence])
external_id = attr.evolve(self.to_entity(db_external_id),
person_external_id_id=None)
person = StatePerson.new_with_defaults(
external_ids=[external_id], sentence_groups=[sentence_group])
expected_person = attr.evolve(self.to_entity(db_person))
expected_person.supervising_officer = new_supervising_officer
expected_supervision_sentence = \
expected_person.sentence_groups[0].supervision_sentences[0]
expected_unchanged_supervision_period = \
attr.evolve(self.to_entity(db_supervision_period))
expected_updated_supervision_period = \
attr.evolve(self.to_entity(db_supervision_period_open),
termination_date=
supervision_period_update.termination_date,
supervising_officer=
expected_unchanged_supervision_period.
supervising_officer)
expected_supervision_sentence.supervision_periods = [
expected_unchanged_supervision_period,
expected_updated_supervision_period, new_supervision_period
]
# Act
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
# Assert
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session)
self.assert_no_errors(matched_entities)
self.assertEqual(1, matched_entities.total_root_entities)
def test_runMatch_supervisingOfficerNotMovedFromPersonOntoOpenSupervisionPeriods(
self):
db_supervising_officer = generate_agent(agent_id=_ID,
external_id=_EXTERNAL_ID,
state_code=_US_MO)
db_person = generate_person(person_id=_ID,
supervising_officer=db_supervising_officer)
db_external_id = generate_external_id(person_external_id_id=_ID,
external_id=_EXTERNAL_ID,
state_code=_US_MO,
id_type=_ID_TYPE)
db_supervision_period = generate_supervision_period(
person=db_person,
supervision_period_id=_ID,
external_id=_EXTERNAL_ID,
start_date=_DATE_1,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
supervising_officer=db_supervising_officer)
db_supervision_period_another = generate_supervision_period(
person=db_person,
supervision_period_id=_ID_2,
external_id=_EXTERNAL_ID_2,
start_date=_DATE_2,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
supervising_officer=db_supervising_officer)
db_closed_supervision_period = generate_supervision_period(
person=db_person,
supervision_period_id=_ID_3,
external_id=_EXTERNAL_ID_3,
start_date=_DATE_3,
termination_date=_DATE_4,
status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
supervising_officer=db_supervising_officer)
db_supervision_sentence = generate_supervision_sentence(
person=db_person,
external_id=_EXTERNAL_ID,
supervision_sentence_id=_ID,
start_date=_DATE_1,
supervision_periods=[
db_supervision_period, db_supervision_period_another,
db_closed_supervision_period
])
db_sentence_group = generate_sentence_group(
external_id=_EXTERNAL_ID,
sentence_group_id=_ID,
supervision_sentences=[db_supervision_sentence])
db_person.external_ids = [db_external_id]
db_person.sentence_groups = [db_sentence_group]
self._commit_to_db(db_person)
external_id = attr.evolve(self.to_entity(db_external_id),
person_external_id_id=None)
new_supervising_officer = StateAgent.new_with_defaults(
external_id=_EXTERNAL_ID_2,
state_code=_US_MO,
agent_type=StateAgentType.SUPERVISION_OFFICER)
person = StatePerson.new_with_defaults(
external_ids=[external_id],
supervising_officer=new_supervising_officer)
expected_person = attr.evolve(self.to_entity(db_person))
# Act 1 - Match
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
# Assert 1 - Match
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session)
self.assert_no_errors(matched_entities)
self.assertEqual(1, matched_entities.total_root_entities)
def test_removeSeosFromSupervisionViolation(self):
supervision_violation_response = \
StateSupervisionViolationResponse.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX)
supervision_violation = StateSupervisionViolation.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX,
supervision_violation_responses=[supervision_violation_response])
placeholder_supervision_period = \
StateSupervisionPeriod.new_with_defaults(
state_code=_US_MO,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
supervision_violation_entries=[supervision_violation])
supervision_sentence = StateSupervisionSentence.new_with_defaults(
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
supervision_periods=[placeholder_supervision_period])
sentence_group = StateSentenceGroup.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
incarceration_sentences=[],
supervision_sentences=[supervision_sentence])
external_id = StatePersonExternalId.new_with_defaults(
state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID)
person = StatePerson.new_with_defaults(
sentence_groups=[sentence_group], external_ids=[external_id])
updated_external_id = _EXTERNAL_ID
expected_supervision_violation_response = attr.evolve(
supervision_violation_response, external_id=updated_external_id)
expected_supervision_violation = attr.evolve(
supervision_violation,
external_id=updated_external_id,
supervision_violation_responses=[
expected_supervision_violation_response
])
expected_placeholder_supervision_period = attr.evolve(
placeholder_supervision_period,
supervision_violation_entries=[expected_supervision_violation])
expected_supervision_sentence = attr.evolve(
supervision_sentence,
supervision_periods=[expected_placeholder_supervision_period])
expected_sentence_group = attr.evolve(
sentence_group,
supervision_sentences=[expected_supervision_sentence])
expected_external_id = attr.evolve(external_id)
expected_person = attr.evolve(
person,
external_ids=[expected_external_id],
sentence_groups=[expected_sentence_group])
# Act 1 - Match
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session)
self.assertEqual(0, matched_entities.error_count)
self.assertEqual(1, matched_entities.total_root_entities)
def test_ssvrFlatFieldMatchingRevocationTypeChanges(self) -> None:
db_person = generate_person(state_code=_US_MO)
db_supervision_violation_response = generate_supervision_violation_response(
person=db_person,
state_code=_US_MO,
response_type=StateSupervisionViolationResponseType.
PERMANENT_DECISION.value,
response_type_raw_text=StateSupervisionViolationResponseType.
PERMANENT_DECISION.value,
deciding_body_type=StateSupervisionViolationResponseDecidingBodyType
.PAROLE_BOARD.value,
deciding_body_type_raw_text=
StateSupervisionViolationResponseDecidingBodyType.PAROLE_BOARD.
value,
revocation_type=StateSupervisionViolationResponseRevocationType.
REINCARCERATION.value,
revocation_type_raw_text="S",
)
db_incarceration_period = generate_incarceration_period(
person=db_person,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION.value,
source_supervision_violation_response=
db_supervision_violation_response,
)
db_incarceration_sentence = generate_incarceration_sentence(
person=db_person,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
incarceration_periods=[db_incarceration_period],
)
db_sentence_group = generate_sentence_group(
person=db_person,
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value,
incarceration_sentences=[db_incarceration_sentence],
supervision_sentences=[],
)
db_external_id = generate_external_id(
person=db_person,
state_code=_US_MO,
id_type=_ID_TYPE,
external_id=_EXTERNAL_ID,
)
db_person.external_ids = [db_external_id]
db_person.sentence_groups = [db_sentence_group]
self._commit_to_db(db_person)
# Even though the revocation type has changed, we still allow a match
supervision_violation_response = StateSupervisionViolationResponse.new_with_defaults(
state_code=_US_MO,
response_type=StateSupervisionViolationResponseType.
PERMANENT_DECISION,
response_type_raw_text=StateSupervisionViolationResponseType.
PERMANENT_DECISION.value,
deciding_body_type=StateSupervisionViolationResponseDecidingBodyType
.PAROLE_BOARD,
deciding_body_type_raw_text=
StateSupervisionViolationResponseDecidingBodyType.PAROLE_BOARD.
value,
revocation_type=StateSupervisionViolationResponseRevocationType.
TREATMENT_IN_PRISON,
revocation_type_raw_text="L",
)
incarceration_period = StateIncarcerationPeriod.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION,
source_supervision_violation_response=
supervision_violation_response,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
incarceration_sentence = StateIncarcerationSentence.new_with_defaults(
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
incarceration_periods=[incarceration_period],
)
sentence_group = StateSentenceGroup.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
incarceration_sentences=[incarceration_sentence],
supervision_sentences=[],
)
external_id = StatePersonExternalId.new_with_defaults(
state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID)
person = StatePerson.new_with_defaults(
sentence_groups=[sentence_group],
external_ids=[external_id],
state_code=_US_MO,
)
expected_person = self.to_entity(db_person)
expected_ip = (expected_person.sentence_groups[0].
incarceration_sentences[0].incarceration_periods[0])
expected_ssvr = expected_ip.source_supervision_violation_response
expected_ssvr.revocation_type = (
StateSupervisionViolationResponseRevocationType.TREATMENT_IN_PRISON
)
expected_ssvr.revocation_type_raw_text = "L"
# Act 1 - Match
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session)
self.assertEqual(0, matched_entities.error_count)
self.assertEqual(1, matched_entities.total_root_entities)
def test_ssvrFlatFieldMatchingWithSomeNullValues(self) -> None:
db_person = generate_person(state_code=_US_MO)
db_supervision_violation_response = generate_supervision_violation_response(
person=db_person,
state_code=_US_MO,
response_type=StateSupervisionViolationResponseType.
PERMANENT_DECISION.value,
response_type_raw_text=StateSupervisionViolationResponseType.
PERMANENT_DECISION.value,
deciding_body_type=StateSupervisionViolationResponseDecidingBodyType
.PAROLE_BOARD.value,
deciding_body_type_raw_text=
StateSupervisionViolationResponseDecidingBodyType.PAROLE_BOARD.
value,
)
db_incarceration_period = generate_incarceration_period(
person=db_person,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION.value,
source_supervision_violation_response=
db_supervision_violation_response,
)
db_incarceration_sentence = generate_incarceration_sentence(
person=db_person,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
incarceration_periods=[db_incarceration_period],
)
db_sentence_group = generate_sentence_group(
person=db_person,
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value,
incarceration_sentences=[db_incarceration_sentence],
supervision_sentences=[],
)
db_external_id = generate_external_id(
person=db_person,
state_code=_US_MO,
id_type=_ID_TYPE,
external_id=_EXTERNAL_ID,
)
db_person.external_ids = [db_external_id]
db_person.sentence_groups = [db_sentence_group]
self._commit_to_db(db_person)
# Even though this violation response doesn't have a deciding_body_type set, it will not clear the values in
# db_supervision_violation_response.
supervision_violation_response = StateSupervisionViolationResponse.new_with_defaults(
state_code=_US_MO,
response_type=StateSupervisionViolationResponseType.
PERMANENT_DECISION,
response_type_raw_text=StateSupervisionViolationResponseType.
PERMANENT_DECISION.value,
)
incarceration_period = StateIncarcerationPeriod.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION,
source_supervision_violation_response=
supervision_violation_response,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
incarceration_sentence = StateIncarcerationSentence.new_with_defaults(
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
state_code=_US_MO,
external_id=_EXTERNAL_ID,
incarceration_periods=[incarceration_period],
)
sentence_group = StateSentenceGroup.new_with_defaults(
state_code=_US_MO,
external_id=_EXTERNAL_ID_WITH_SUFFIX,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
incarceration_sentences=[incarceration_sentence],
supervision_sentences=[],
)
external_id = StatePersonExternalId.new_with_defaults(
state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID)
person = StatePerson.new_with_defaults(
sentence_groups=[sentence_group],
external_ids=[external_id],
state_code=_US_MO,
)
expected_person = self.to_entity(db_person)
# Act 1 - Match
session = self._session()
matched_entities = entity_matching.match(session,
_US_MO,
ingested_people=[person])
self.assert_people_match_pre_and_post_commit([expected_person],
matched_entities.people,
session)
self.assertEqual(0, matched_entities.error_count)
self.assertEqual(1, matched_entities.total_root_entities)