def match_and_write_people(session: Session) -> bool: logging.info("Starting entity matching") entity_matching_output = entity_matching.match( session, metadata.region, people) output_people = entity_matching_output.people total_root_entities = total_people \ if metadata.system_level == SystemLevel.COUNTY \ else entity_matching_output.total_root_entities logging.info("Completed entity matching with [%s] errors", entity_matching_output.error_count) logging.info( "Completed entity matching and have [%s] total people " "to commit to DB", len(output_people)) if _should_abort( total_root_entities=total_root_entities, conversion_result=conversion_result, entity_matching_errors=entity_matching_output.error_count, data_validation_errors=data_validation_errors): # TODO(#1665): remove once dangling PERSIST session # investigation is complete. logging.info("_should_abort_ was true after entity matching") return False database.write_people( session, output_people, metadata, orphaned_entities=entity_matching_output.orphaned_entities) logging.info("Successfully wrote to the database") return True
def test_matchPerson_updateStatusOnOrphanedEntities(self): # Arrange schema_bond = schema.Bond( bond_id=_BOND_ID, status=BondStatus.PENDING.value, booking_id=_BOOKING_ID) schema_charge = schema.Charge( charge_id=_CHARGE_ID, status=ChargeStatus.PENDING.value, bond=schema_bond) schema_booking = schema.Booking( admission_date=_DATE_2, booking_id=_BOOKING_ID, custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE, first_seen_time=_DATE, charges=[schema_charge]) schema_person = schema.Person( person_id=_PERSON_ID, full_name=_FULL_NAME, birthdate=_DATE, jurisdiction_id=_JURISDICTION_ID, region=_REGION, bookings=[schema_booking]) session = SessionFactory.for_schema_base(JailsBase) session.add(schema_person) session.commit() ingested_charge_no_bond = attr.evolve( converter.convert_schema_object_to_entity(schema_charge), charge_id=None, bond=None) ingested_booking = attr.evolve( converter.convert_schema_object_to_entity(schema_booking), booking_id=None, custody_status=CustodyStatus.RELEASED, charges=[ingested_charge_no_bond]) ingested_person = attr.evolve( converter.convert_schema_object_to_entity(schema_person), person_id=None, bookings=[ingested_booking]) # Act out = entity_matching.match(session, _REGION, [ingested_person]) # Assert expected_orphaned_bond = attr.evolve( converter.convert_schema_object_to_entity(schema_bond), status=BondStatus.REMOVED_WITHOUT_INFO) expected_charge = attr.evolve( ingested_charge_no_bond, charge_id=schema_charge.charge_id) expected_booking = attr.evolve( ingested_booking, booking_id=schema_booking.booking_id, charges=[expected_charge]) expected_person = attr.evolve( ingested_person, person_id=schema_person.person_id, bookings=[expected_booking]) self.assertCountEqual( converter.convert_schema_objects_to_entity(out.people), [expected_person]) self.assertCountEqual( converter.convert_schema_objects_to_entity(out.orphaned_entities), [expected_orphaned_bond]) self.assertEqual(out.error_count, 0)
def test_matchPeople_errorCount(self): # Arrange schema_booking = schema.Booking( external_id=_EXTERNAL_ID, admission_date=_DATE_2, booking_id=_BOOKING_ID, custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE, first_seen_time=_DATE) schema_booking_another = copy.deepcopy(schema_booking) schema_booking_another.booking_id = _BOOKING_ID_ANOTHER schema_person = schema.Person( person_id=_PERSON_ID, external_id=_EXTERNAL_ID, jurisdiction_id=_JURISDICTION_ID, full_name=_FULL_NAME, birthdate=_DATE, region=_REGION, bookings=[schema_booking, schema_booking_another]) schema_person_another = schema.Person(person_id=_PERSON_ID_ANOTHER, jurisdiction_id=_JURISDICTION_ID, region=_REGION, full_name=_NAME_2, external_id=_EXTERNAL_ID_ANOTHER) session = SessionFactory.for_schema_base(JailsBase) session.add(schema_person) session.add(schema_person_another) session.commit() ingested_booking = attr.evolve( converter.convert_schema_object_to_entity(schema_booking), booking_id=None, custody_status=CustodyStatus.RELEASED) ingested_person = attr.evolve( converter.convert_schema_object_to_entity(schema_person), person_id=None, bookings=[ingested_booking]) ingested_person_another = attr.evolve( converter.convert_schema_object_to_entity(schema_person_another), person_id=None ) # Act out = entity_matching.match( session, _REGION, [ingested_person, ingested_person_another]) # Assert expected_person = attr.evolve(ingested_person_another, person_id=schema_person_another.person_id) self.assertCountEqual( converter.convert_schema_objects_to_entity(out.people), [expected_person]) self.assertCountEqual( converter.convert_schema_objects_to_entity(out.orphaned_entities), []) self.assertEqual(out.error_count, 1)
def test_matchPeople_twoMatchingPeople_PicksMostSimilar(self): # Arrange schema_person = schema.Person( person_id=_PERSON_ID, external_id=_EXTERNAL_ID, jurisdiction_id=_JURISDICTION_ID, full_name=_FULL_NAME, birthdate=_DATE, region=_REGION, gender=Gender.MALE.value, ) schema_person_mismatch = copy.deepcopy(schema_person) schema_person_mismatch.person_id = _PERSON_ID_ANOTHER schema_person_mismatch.gender = Gender.FEMALE.value session = SessionFactory.for_schema_base(JailsBase) session.add(schema_person) session.add(schema_person_mismatch) session.commit() ingested_person = attr.evolve( converter.convert_schema_object_to_entity(schema_person), person_id=None ) expected_person = attr.evolve( ingested_person, person_id=schema_person.person_id ) # Act matched_entities = entity_matching.match(session, _REGION, [ingested_person]) # Assert both schema objects are matches, but we select the most # similar one. self.assertTrue( county_matching_utils.is_person_match( db_entity=schema_person, ingested_entity=ingested_person ) ) self.assertTrue( county_matching_utils.is_person_match( db_entity=schema_person_mismatch, ingested_entity=ingested_person ) ) self.assertEqual(matched_entities.error_count, 0) self.assertEqual(len(matched_entities.orphaned_entities), 0) self.assertEqual(ingested_person, expected_person)
def test_matchPeople(self): # Arrange schema_booking = schema.Booking( admission_date=_DATE_2, booking_id=_BOOKING_ID, custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE, first_seen_time=_DATE, ) schema_person = schema.Person( person_id=_PERSON_ID, full_name=_FULL_NAME, birthdate=_DATE, jurisdiction_id=_JURISDICTION_ID, region=_REGION, bookings=[schema_booking], ) schema_booking_external_id = schema.Booking( admission_date=_DATE_2, booking_id=_BOOKING_ID_ANOTHER, release_date=_DATE, custody_status=CustodyStatus.RELEASED.value, last_seen_time=_DATE, first_seen_time=_DATE, ) schema_person_external_id = schema.Person( person_id=_PERSON_ID_ANOTHER, external_id=_EXTERNAL_ID, full_name=_FULL_NAME, birthdate=_DATE, jurisdiction_id=_JURISDICTION_ID, region=_REGION, bookings=[schema_booking_external_id], ) with SessionFactory.using_database(self.database_key, autocommit=False) as session: session.add(schema_person) session.add(schema_person_external_id) session.commit() ingested_booking = attr.evolve( converter.convert_schema_object_to_entity(schema_booking), booking_id=None, custody_status=CustodyStatus.RELEASED, ) ingested_person = attr.evolve( converter.convert_schema_object_to_entity(schema_person), person_id=None, bookings=[ingested_booking], ) ingested_booking_external_id = attr.evolve( converter.convert_schema_object_to_entity( schema_booking_external_id), booking_id=None, facility=_FACILITY, ) ingested_person_external_id = attr.evolve( converter.convert_schema_object_to_entity( schema_person_external_id), person_id=None, bookings=[ingested_booking_external_id], ) # Act out = entity_matching.match( session, _REGION, [ingested_person_external_id, ingested_person]) # Assert expected_booking = attr.evolve(ingested_booking, booking_id=_BOOKING_ID) expected_person = attr.evolve(ingested_person, person_id=_PERSON_ID, bookings=[expected_booking]) expected_booking_external_id = attr.evolve( ingested_booking_external_id, booking_id=_BOOKING_ID_ANOTHER) expected_person_external_id = attr.evolve( ingested_person_external_id, person_id=_PERSON_ID_ANOTHER, bookings=[expected_booking_external_id], ) self.assertCountEqual( converter.convert_schema_objects_to_entity(out.people), [expected_person_external_id, expected_person], ) self.assertCountEqual( converter.convert_schema_objects_to_entity(out.orphaned_entities), []) self.assertEqual(out.error_count, 0)
def write(ingest_info, metadata): """ If in prod or if 'PERSIST_LOCALLY' is set to true, persist each person in the ingest_info. If a person with the given surname/birthday already exists, then update that person. Otherwise, simply log the given ingest_infos for debugging """ ingest_info_validator.validate(ingest_info) mtags = { monitoring.TagKey.SHOULD_PERSIST: _should_persist(), monitoring.TagKey.PERSISTED: False } total_people = _get_total_people(ingest_info, metadata) with monitoring.measurements(mtags) as measurements: # Convert the people one at a time and count the errors as they happen. conversion_result: IngestInfoConversionResult = \ ingest_info_converter.convert_to_persistence_entities(ingest_info, metadata) people, data_validation_errors = entity_validator.validate( conversion_result.people) logging.info( "Converted [%s] people with [%s] enum_parsing_errors, [%s]" " general_parsing_errors, [%s] protected_class_errors and " "[%s] data_validation_errors", len(people), conversion_result.enum_parsing_errors, conversion_result.general_parsing_errors, conversion_result.protected_class_errors, data_validation_errors) measurements.measure_int_put(m_people, len(people)) if _should_abort(total_root_entities=total_people, conversion_result=conversion_result, data_validation_errors=data_validation_errors): # TODO(#1665): remove once dangling PERSIST session investigation # is complete. logging.info("_should_abort_ was true after converting people") return False if not _should_persist(): return True persisted = False session = SessionFactory.for_schema_base( schema_base_for_system_level(metadata.system_level)) try: logging.info("Starting entity matching") entity_matching_output = entity_matching.match( session, metadata.region, people) people = entity_matching_output.people total_root_entities = total_people \ if metadata.system_level == SystemLevel.COUNTY \ else entity_matching_output.total_root_entities logging.info("Completed entity matching with [%s] errors", entity_matching_output.error_count) logging.info( "Completed entity matching and have [%s] total people " "to commit to DB", len(people)) if _should_abort( total_root_entities=total_root_entities, conversion_result=conversion_result, entity_matching_errors=entity_matching_output.error_count, data_validation_errors=data_validation_errors): # TODO(#1665): remove once dangling PERSIST session # investigation is complete. logging.info("_should_abort_ was true after entity matching") return False database.write_people( session, people, metadata, orphaned_entities=entity_matching_output.orphaned_entities) logging.info("Successfully wrote to the database") session.commit() persisted = True mtags[monitoring.TagKey.PERSISTED] = True except Exception as e: logging.exception("An exception was raised in write(): [%s]", type(e).__name__) # Record the error type that happened and increment the counter mtags[monitoring.TagKey.ERROR] = type(e).__name__ measurements.measure_int_put(m_errors, 1) session.rollback() raise finally: session.close() return persisted
def test_supervisionViolationsWithDifferentParents_mergesViolations(self): db_person = generate_person(person_id=_ID, full_name=_FULL_NAME) db_supervision_violation = generate_supervision_violation( person=db_person, state_code=_US_MO, supervision_violation_id=_ID, external_id=_EXTERNAL_ID) db_placeholder_supervision_period = generate_supervision_period( person=db_person, state_code=_US_MO, supervision_period_id=_ID, supervision_violation_entries=[db_supervision_violation]) db_incarceration_sentence = generate_incarceration_sentence( person=db_person, state_code=_US_MO, incarceration_sentence_id=_ID, external_id=_EXTERNAL_ID, supervision_periods=[db_placeholder_supervision_period]) db_sentence_group = generate_sentence_group( person=db_person, state_code=_US_MO, sentence_group_id=_ID, external_id=_EXTERNAL_ID, incarceration_sentences=[db_incarceration_sentence]) db_external_id = generate_external_id(person=db_person, person_external_id_id=_ID, state_code=_US_MO, external_id=_EXTERNAL_ID) db_person.sentence_groups = [db_sentence_group] db_person.external_ids = [db_external_id] self._commit_to_db(db_person) supervision_violation = attr.evolve( self.to_entity(db_supervision_violation), supervision_violation_id=None, external_id=_EXTERNAL_ID_WITH_SUFFIX) placeholder_supervision_period = attr.evolve( self.to_entity(db_placeholder_supervision_period), supervision_period_id=None, supervision_violation_entries=[supervision_violation]) supervision_sentence = StateSupervisionSentence.new_with_defaults( status=StateSentenceStatus.PRESENT_WITHOUT_INFO, state_code=_US_MO, external_id=_EXTERNAL_ID, supervision_periods=[placeholder_supervision_period]) sentence_group = attr.evolve( self.to_entity(db_sentence_group), sentence_group_id=None, incarceration_sentences=[], supervision_sentences=[supervision_sentence]) external_id = attr.evolve(self.to_entity(db_external_id), person_external_id_id=None) person = attr.evolve(self.to_entity(db_person), person_id=None, sentence_groups=[sentence_group], external_ids=[external_id]) expected_supervision_violation = attr.evolve( self.to_entity(db_supervision_violation)) expected_placeholder_supervision_period_is = attr.evolve( placeholder_supervision_period, supervision_violation_entries=[expected_supervision_violation]) expected_placeholder_supervision_period_ss = attr.evolve( self.to_entity(db_placeholder_supervision_period), supervision_violation_entries=[expected_supervision_violation]) expected_supervision_sentence = attr.evolve( supervision_sentence, supervision_periods=[expected_placeholder_supervision_period_ss]) expected_incarceration_sentence = attr.evolve( self.to_entity(db_incarceration_sentence), supervision_periods=[expected_placeholder_supervision_period_is]) expected_sentence_group = attr.evolve( self.to_entity(db_sentence_group), incarceration_sentences=[expected_incarceration_sentence], supervision_sentences=[expected_supervision_sentence]) expected_external_id = attr.evolve(self.to_entity(db_external_id)) expected_person = attr.evolve( self.to_entity(db_person), external_ids=[expected_external_id], sentence_groups=[expected_sentence_group]) # Act 1 - Match session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session) self.assertEqual(0, matched_entities.error_count) self.assertEqual(1, matched_entities.total_root_entities)
def test_runMatch_supervisionPeriodDateChangesSoItDoesNotMatchSentenceOrViolations( self): # Arrange db_supervising_officer = generate_agent( agent_id=_ID, external_id=_EXTERNAL_ID, state_code=_US_MO, agent_type=StateAgentType.SUPERVISION_OFFICER.value) db_person = generate_person(person_id=_ID, supervising_officer=db_supervising_officer) db_external_id = generate_external_id(person_external_id_id=_ID, external_id=_EXTERNAL_ID, state_code=_US_MO, id_type=_ID_TYPE) # Violation has been date matched to the open supervision period db_supervision_violation = generate_supervision_violation( person=db_person, state_code=_US_MO, supervision_violation_id=_ID, external_id=_EXTERNAL_ID, violation_date=_DATE_4) db_supervision_period_open = generate_supervision_period( person=db_person, supervision_period_id=_ID_2, external_id=_EXTERNAL_ID_2, start_date=_DATE_2, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, supervising_officer=db_supervising_officer, supervision_violation_entries=[db_supervision_violation]) db_supervision_sentence = generate_supervision_sentence( person=db_person, external_id=_EXTERNAL_ID, supervision_sentence_id=_ID, state_code=_US_MO, start_date=_DATE_1, supervision_periods=[db_supervision_period_open]) db_sentence_group = generate_sentence_group( external_id=_EXTERNAL_ID, sentence_group_id=_ID, state_code=_US_MO, supervision_sentences=[db_supervision_sentence]) db_person.external_ids = [db_external_id] db_person.sentence_groups = [db_sentence_group] self._commit_to_db(db_person) supervsion_period_updated = StateSupervisionPeriod.new_with_defaults( state_code=_US_MO, external_id=db_supervision_period_open.external_id, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO, start_date=_DATE_2, termination_date=_DATE_3, ) placeholder_supervision_sentence = StateSupervisionSentence.new_with_defaults( state_code=_US_MO, supervision_periods=[supervsion_period_updated]) sentence_group = StateSentenceGroup.new_with_defaults( external_id=db_sentence_group.external_id, state_code=_US_MO, supervision_sentences=[placeholder_supervision_sentence]) external_id = StatePersonExternalId.new_with_defaults( external_id=db_external_id.external_id, state_code=_US_MO, id_type=db_external_id.id_type) person = StatePerson.new_with_defaults( external_ids=[external_id], sentence_groups=[sentence_group]) expected_person = attr.evolve(self.to_entity(db_person)) expected_sentence = expected_person.sentence_groups[ 0].supervision_sentences[0] expected_original_supervision_period = expected_sentence.supervision_periods[ 0] # Violation is moved off of the supervision period (it no longer matches) and the termination date is updated expected_original_supervision_period.supervision_violation_entries = [] expected_original_supervision_period.termination_date = _DATE_3 # A placeholder periods is created to hold the existing supervision violation expected_new_placeholder_supervision_period = StateSupervisionPeriod.new_with_defaults( state_code=_US_MO, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO, supervision_violation_entries=[ self.to_entity(db_supervision_violation) ]) expected_sentence.supervision_periods.append( expected_new_placeholder_supervision_period) # Act session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) # Assert self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session, debug=True) self.assert_no_errors(matched_entities) self.assertEqual(1, matched_entities.total_root_entities)
def test_runMatch_supervisingOfficerMovedFromSupervisionPeriodToPerson( self): # Arrange db_supervising_officer = generate_agent( agent_id=_ID, external_id=_EXTERNAL_ID, state_code=_US_MO, agent_type=StateAgentType.SUPERVISION_OFFICER.value) db_person = generate_person(person_id=_ID, supervising_officer=db_supervising_officer) db_external_id = generate_external_id(person_external_id_id=_ID, external_id=_EXTERNAL_ID, state_code=_US_MO, id_type=_ID_TYPE) db_supervision_period = generate_supervision_period( person=db_person, supervision_period_id=_ID, external_id=_EXTERNAL_ID, start_date=_DATE_1, termination_date=_DATE_2, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, supervising_officer=db_supervising_officer) db_supervision_period_open = generate_supervision_period( person=db_person, supervision_period_id=_ID_2, external_id=_EXTERNAL_ID_2, start_date=_DATE_2, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, supervising_officer=db_supervising_officer) db_supervision_sentence = generate_supervision_sentence( person=db_person, external_id=_EXTERNAL_ID, supervision_sentence_id=_ID, state_code=_US_MO, start_date=_DATE_1, supervision_periods=[ db_supervision_period, db_supervision_period_open ]) db_sentence_group = generate_sentence_group( external_id=_EXTERNAL_ID, sentence_group_id=_ID, state_code=_US_MO, supervision_sentences=[db_supervision_sentence]) db_person.external_ids = [db_external_id] db_person.sentence_groups = [db_sentence_group] self._commit_to_db(db_person) new_supervising_officer = StateAgent.new_with_defaults( external_id=_EXTERNAL_ID_2, state_code=_US_MO, agent_type=StateAgentType.SUPERVISION_OFFICER) new_supervision_period = StateSupervisionPeriod.new_with_defaults( external_id=_EXTERNAL_ID_3, state_code=_US_MO, start_date=_DATE_3, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO, supervising_officer=new_supervising_officer) supervision_period_update = StateSupervisionPeriod.new_with_defaults( external_id=db_supervision_period_open.external_id, state_code=_US_MO, termination_date=_DATE_3) supervision_sentence = StateSupervisionSentence.new_with_defaults( external_id=db_supervision_sentence.external_id, state_code=_US_MO, supervision_periods=[ supervision_period_update, new_supervision_period ]) sentence_group = StateSentenceGroup.new_with_defaults( external_id=db_sentence_group.external_id, state_code=_US_MO, supervision_sentences=[supervision_sentence]) external_id = attr.evolve(self.to_entity(db_external_id), person_external_id_id=None) person = StatePerson.new_with_defaults( external_ids=[external_id], sentence_groups=[sentence_group]) expected_person = attr.evolve(self.to_entity(db_person)) expected_person.supervising_officer = new_supervising_officer expected_supervision_sentence = \ expected_person.sentence_groups[0].supervision_sentences[0] expected_unchanged_supervision_period = \ attr.evolve(self.to_entity(db_supervision_period)) expected_updated_supervision_period = \ attr.evolve(self.to_entity(db_supervision_period_open), termination_date= supervision_period_update.termination_date, supervising_officer= expected_unchanged_supervision_period. supervising_officer) expected_supervision_sentence.supervision_periods = [ expected_unchanged_supervision_period, expected_updated_supervision_period, new_supervision_period ] # Act session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) # Assert self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session) self.assert_no_errors(matched_entities) self.assertEqual(1, matched_entities.total_root_entities)
def test_runMatch_supervisingOfficerNotMovedFromPersonOntoOpenSupervisionPeriods( self): db_supervising_officer = generate_agent(agent_id=_ID, external_id=_EXTERNAL_ID, state_code=_US_MO) db_person = generate_person(person_id=_ID, supervising_officer=db_supervising_officer) db_external_id = generate_external_id(person_external_id_id=_ID, external_id=_EXTERNAL_ID, state_code=_US_MO, id_type=_ID_TYPE) db_supervision_period = generate_supervision_period( person=db_person, supervision_period_id=_ID, external_id=_EXTERNAL_ID, start_date=_DATE_1, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, supervising_officer=db_supervising_officer) db_supervision_period_another = generate_supervision_period( person=db_person, supervision_period_id=_ID_2, external_id=_EXTERNAL_ID_2, start_date=_DATE_2, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, supervising_officer=db_supervising_officer) db_closed_supervision_period = generate_supervision_period( person=db_person, supervision_period_id=_ID_3, external_id=_EXTERNAL_ID_3, start_date=_DATE_3, termination_date=_DATE_4, status=StateSupervisionPeriodStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, supervising_officer=db_supervising_officer) db_supervision_sentence = generate_supervision_sentence( person=db_person, external_id=_EXTERNAL_ID, supervision_sentence_id=_ID, start_date=_DATE_1, supervision_periods=[ db_supervision_period, db_supervision_period_another, db_closed_supervision_period ]) db_sentence_group = generate_sentence_group( external_id=_EXTERNAL_ID, sentence_group_id=_ID, supervision_sentences=[db_supervision_sentence]) db_person.external_ids = [db_external_id] db_person.sentence_groups = [db_sentence_group] self._commit_to_db(db_person) external_id = attr.evolve(self.to_entity(db_external_id), person_external_id_id=None) new_supervising_officer = StateAgent.new_with_defaults( external_id=_EXTERNAL_ID_2, state_code=_US_MO, agent_type=StateAgentType.SUPERVISION_OFFICER) person = StatePerson.new_with_defaults( external_ids=[external_id], supervising_officer=new_supervising_officer) expected_person = attr.evolve(self.to_entity(db_person)) # Act 1 - Match session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) # Assert 1 - Match self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session) self.assert_no_errors(matched_entities) self.assertEqual(1, matched_entities.total_root_entities)
def test_removeSeosFromSupervisionViolation(self): supervision_violation_response = \ StateSupervisionViolationResponse.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX) supervision_violation = StateSupervisionViolation.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX, supervision_violation_responses=[supervision_violation_response]) placeholder_supervision_period = \ StateSupervisionPeriod.new_with_defaults( state_code=_US_MO, status=StateSentenceStatus.PRESENT_WITHOUT_INFO, supervision_violation_entries=[supervision_violation]) supervision_sentence = StateSupervisionSentence.new_with_defaults( status=StateSentenceStatus.PRESENT_WITHOUT_INFO, state_code=_US_MO, external_id=_EXTERNAL_ID, supervision_periods=[placeholder_supervision_period]) sentence_group = StateSentenceGroup.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX, status=StateSentenceStatus.PRESENT_WITHOUT_INFO, incarceration_sentences=[], supervision_sentences=[supervision_sentence]) external_id = StatePersonExternalId.new_with_defaults( state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID) person = StatePerson.new_with_defaults( sentence_groups=[sentence_group], external_ids=[external_id]) updated_external_id = _EXTERNAL_ID expected_supervision_violation_response = attr.evolve( supervision_violation_response, external_id=updated_external_id) expected_supervision_violation = attr.evolve( supervision_violation, external_id=updated_external_id, supervision_violation_responses=[ expected_supervision_violation_response ]) expected_placeholder_supervision_period = attr.evolve( placeholder_supervision_period, supervision_violation_entries=[expected_supervision_violation]) expected_supervision_sentence = attr.evolve( supervision_sentence, supervision_periods=[expected_placeholder_supervision_period]) expected_sentence_group = attr.evolve( sentence_group, supervision_sentences=[expected_supervision_sentence]) expected_external_id = attr.evolve(external_id) expected_person = attr.evolve( person, external_ids=[expected_external_id], sentence_groups=[expected_sentence_group]) # Act 1 - Match session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session) self.assertEqual(0, matched_entities.error_count) self.assertEqual(1, matched_entities.total_root_entities)
def test_ssvrFlatFieldMatchingRevocationTypeChanges(self) -> None: db_person = generate_person(state_code=_US_MO) db_supervision_violation_response = generate_supervision_violation_response( person=db_person, state_code=_US_MO, response_type=StateSupervisionViolationResponseType. PERMANENT_DECISION.value, response_type_raw_text=StateSupervisionViolationResponseType. PERMANENT_DECISION.value, deciding_body_type=StateSupervisionViolationResponseDecidingBodyType .PAROLE_BOARD.value, deciding_body_type_raw_text= StateSupervisionViolationResponseDecidingBodyType.PAROLE_BOARD. value, revocation_type=StateSupervisionViolationResponseRevocationType. REINCARCERATION.value, revocation_type_raw_text="S", ) db_incarceration_period = generate_incarceration_period( person=db_person, state_code=_US_MO, external_id=_EXTERNAL_ID, admission_reason=StateIncarcerationPeriodAdmissionReason. PAROLE_REVOCATION.value, source_supervision_violation_response= db_supervision_violation_response, ) db_incarceration_sentence = generate_incarceration_sentence( person=db_person, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, external_id=_EXTERNAL_ID, incarceration_periods=[db_incarceration_period], ) db_sentence_group = generate_sentence_group( person=db_person, state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, incarceration_sentences=[db_incarceration_sentence], supervision_sentences=[], ) db_external_id = generate_external_id( person=db_person, state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID, ) db_person.external_ids = [db_external_id] db_person.sentence_groups = [db_sentence_group] self._commit_to_db(db_person) # Even though the revocation type has changed, we still allow a match supervision_violation_response = StateSupervisionViolationResponse.new_with_defaults( state_code=_US_MO, response_type=StateSupervisionViolationResponseType. PERMANENT_DECISION, response_type_raw_text=StateSupervisionViolationResponseType. PERMANENT_DECISION.value, deciding_body_type=StateSupervisionViolationResponseDecidingBodyType .PAROLE_BOARD, deciding_body_type_raw_text= StateSupervisionViolationResponseDecidingBodyType.PAROLE_BOARD. value, revocation_type=StateSupervisionViolationResponseRevocationType. TREATMENT_IN_PRISON, revocation_type_raw_text="L", ) incarceration_period = StateIncarcerationPeriod.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID, admission_reason=StateIncarcerationPeriodAdmissionReason. PAROLE_REVOCATION, source_supervision_violation_response= supervision_violation_response, status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO, ) incarceration_sentence = StateIncarcerationSentence.new_with_defaults( status=StateSentenceStatus.PRESENT_WITHOUT_INFO, state_code=_US_MO, external_id=_EXTERNAL_ID, incarceration_periods=[incarceration_period], ) sentence_group = StateSentenceGroup.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX, status=StateSentenceStatus.PRESENT_WITHOUT_INFO, incarceration_sentences=[incarceration_sentence], supervision_sentences=[], ) external_id = StatePersonExternalId.new_with_defaults( state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID) person = StatePerson.new_with_defaults( sentence_groups=[sentence_group], external_ids=[external_id], state_code=_US_MO, ) expected_person = self.to_entity(db_person) expected_ip = (expected_person.sentence_groups[0]. incarceration_sentences[0].incarceration_periods[0]) expected_ssvr = expected_ip.source_supervision_violation_response expected_ssvr.revocation_type = ( StateSupervisionViolationResponseRevocationType.TREATMENT_IN_PRISON ) expected_ssvr.revocation_type_raw_text = "L" # Act 1 - Match session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session) self.assertEqual(0, matched_entities.error_count) self.assertEqual(1, matched_entities.total_root_entities)
def test_ssvrFlatFieldMatchingWithSomeNullValues(self) -> None: db_person = generate_person(state_code=_US_MO) db_supervision_violation_response = generate_supervision_violation_response( person=db_person, state_code=_US_MO, response_type=StateSupervisionViolationResponseType. PERMANENT_DECISION.value, response_type_raw_text=StateSupervisionViolationResponseType. PERMANENT_DECISION.value, deciding_body_type=StateSupervisionViolationResponseDecidingBodyType .PAROLE_BOARD.value, deciding_body_type_raw_text= StateSupervisionViolationResponseDecidingBodyType.PAROLE_BOARD. value, ) db_incarceration_period = generate_incarceration_period( person=db_person, state_code=_US_MO, external_id=_EXTERNAL_ID, admission_reason=StateIncarcerationPeriodAdmissionReason. PAROLE_REVOCATION.value, source_supervision_violation_response= db_supervision_violation_response, ) db_incarceration_sentence = generate_incarceration_sentence( person=db_person, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, state_code=_US_MO, external_id=_EXTERNAL_ID, incarceration_periods=[db_incarceration_period], ) db_sentence_group = generate_sentence_group( person=db_person, state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX, status=StateSentenceStatus.PRESENT_WITHOUT_INFO.value, incarceration_sentences=[db_incarceration_sentence], supervision_sentences=[], ) db_external_id = generate_external_id( person=db_person, state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID, ) db_person.external_ids = [db_external_id] db_person.sentence_groups = [db_sentence_group] self._commit_to_db(db_person) # Even though this violation response doesn't have a deciding_body_type set, it will not clear the values in # db_supervision_violation_response. supervision_violation_response = StateSupervisionViolationResponse.new_with_defaults( state_code=_US_MO, response_type=StateSupervisionViolationResponseType. PERMANENT_DECISION, response_type_raw_text=StateSupervisionViolationResponseType. PERMANENT_DECISION.value, ) incarceration_period = StateIncarcerationPeriod.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID, admission_reason=StateIncarcerationPeriodAdmissionReason. PAROLE_REVOCATION, source_supervision_violation_response= supervision_violation_response, status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO, ) incarceration_sentence = StateIncarcerationSentence.new_with_defaults( status=StateSentenceStatus.PRESENT_WITHOUT_INFO, state_code=_US_MO, external_id=_EXTERNAL_ID, incarceration_periods=[incarceration_period], ) sentence_group = StateSentenceGroup.new_with_defaults( state_code=_US_MO, external_id=_EXTERNAL_ID_WITH_SUFFIX, status=StateSentenceStatus.PRESENT_WITHOUT_INFO, incarceration_sentences=[incarceration_sentence], supervision_sentences=[], ) external_id = StatePersonExternalId.new_with_defaults( state_code=_US_MO, id_type=_ID_TYPE, external_id=_EXTERNAL_ID) person = StatePerson.new_with_defaults( sentence_groups=[sentence_group], external_ids=[external_id], state_code=_US_MO, ) expected_person = self.to_entity(db_person) # Act 1 - Match session = self._session() matched_entities = entity_matching.match(session, _US_MO, ingested_people=[person]) self.assert_people_match_pre_and_post_commit([expected_person], matched_entities.people, session) self.assertEqual(0, matched_entities.error_count) self.assertEqual(1, matched_entities.total_root_entities)