def test_matchPerson_updateStatusOnOrphanedEntities(self):
# Arrange
schema_bond = schema.Bond(
bond_id=_BOND_ID, status=BondStatus.PENDING.value,
booking_id=_BOOKING_ID)
schema_charge = schema.Charge(
charge_id=_CHARGE_ID, status=ChargeStatus.PENDING.value,
bond=schema_bond)
schema_booking = schema.Booking(
admission_date=_DATE_2, booking_id=_BOOKING_ID,
custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE,
first_seen_time=_DATE, charges=[schema_charge])
schema_person = schema.Person(
person_id=_PERSON_ID, full_name=_FULL_NAME, birthdate=_DATE,
jurisdiction_id=_JURISDICTION_ID, region=_REGION,
bookings=[schema_booking])
session = SessionFactory.for_schema_base(JailsBase)
session.add(schema_person)
session.commit()
ingested_charge_no_bond = attr.evolve(
converter.convert_schema_object_to_entity(schema_charge),
charge_id=None,
bond=None)
ingested_booking = attr.evolve(
converter.convert_schema_object_to_entity(schema_booking),
booking_id=None,
custody_status=CustodyStatus.RELEASED,
charges=[ingested_charge_no_bond])
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person),
person_id=None,
bookings=[ingested_booking])
# Act
out = entity_matching.match(session, _REGION, [ingested_person])
# Assert
expected_orphaned_bond = attr.evolve(
converter.convert_schema_object_to_entity(schema_bond),
status=BondStatus.REMOVED_WITHOUT_INFO)
expected_charge = attr.evolve(
ingested_charge_no_bond, charge_id=schema_charge.charge_id)
expected_booking = attr.evolve(
ingested_booking, booking_id=schema_booking.booking_id,
charges=[expected_charge])
expected_person = attr.evolve(
ingested_person, person_id=schema_person.person_id,
bookings=[expected_booking])
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.people),
[expected_person])
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.orphaned_entities),
[expected_orphaned_bond])
self.assertEqual(out.error_count, 0)
def test_matchPeople_errorCount(self):
# Arrange
schema_booking = schema.Booking(
external_id=_EXTERNAL_ID, admission_date=_DATE_2,
booking_id=_BOOKING_ID,
custody_status=CustodyStatus.IN_CUSTODY.value, last_seen_time=_DATE,
first_seen_time=_DATE)
schema_booking_another = copy.deepcopy(schema_booking)
schema_booking_another.booking_id = _BOOKING_ID_ANOTHER
schema_person = schema.Person(
person_id=_PERSON_ID, external_id=_EXTERNAL_ID,
jurisdiction_id=_JURISDICTION_ID,
full_name=_FULL_NAME, birthdate=_DATE,
region=_REGION, bookings=[schema_booking, schema_booking_another])
schema_person_another = schema.Person(person_id=_PERSON_ID_ANOTHER,
jurisdiction_id=_JURISDICTION_ID,
region=_REGION,
full_name=_NAME_2,
external_id=_EXTERNAL_ID_ANOTHER)
session = SessionFactory.for_schema_base(JailsBase)
session.add(schema_person)
session.add(schema_person_another)
session.commit()
ingested_booking = attr.evolve(
converter.convert_schema_object_to_entity(schema_booking),
booking_id=None,
custody_status=CustodyStatus.RELEASED)
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person),
person_id=None,
bookings=[ingested_booking])
ingested_person_another = attr.evolve(
converter.convert_schema_object_to_entity(schema_person_another),
person_id=None
)
# Act
out = entity_matching.match(
session, _REGION, [ingested_person, ingested_person_another])
# Assert
expected_person = attr.evolve(ingested_person_another,
person_id=schema_person_another.person_id)
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.people),
[expected_person])
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.orphaned_entities),
[])
self.assertEqual(out.error_count, 1)
def assert_people_match(self, expected_people: List[StatePerson],
matched_people: List[schema.StatePerson]):
converted_matched = \
converter.convert_schema_objects_to_entity(matched_people)
db_expected_with_backedges = \
converter.convert_entity_people_to_schema_people(expected_people)
expected_with_backedges = \
converter.convert_schema_objects_to_entity(
db_expected_with_backedges)
self.assertEqual(expected_with_backedges, converted_matched)
def test_getFieldAsList(self) -> None:
fine = entities.StateFine.new_with_defaults(
state_code="us_nc",
external_id="ex1",
status=StateFineStatus.PRESENT_WITHOUT_INFO,
)
fine_2 = entities.StateFine.new_with_defaults(
state_code="us_nc",
external_id="ex2",
status=StateFineStatus.PRESENT_WITHOUT_INFO,
)
entity = entities.StateSentenceGroup.new_with_defaults(
state_code="us_nc",
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
fines=[fine, fine_2],
)
db_entity = converter.convert_entity_to_schema_object(entity)
self.assertCountEqual(["us_nc"],
entity.get_field_as_list("state_code"))
self.assertCountEqual([fine, fine_2],
entity.get_field_as_list("fines"))
self.assertCountEqual(
[], entity.get_field_as_list("supervision_sentences"))
self.assertCountEqual(["us_nc"],
db_entity.get_field_as_list("state_code"))
self.assertCountEqual(
[fine, fine_2],
converter.convert_schema_objects_to_entity(
db_entity.get_field_as_list("fines"),
populate_back_edges=False),
)
self.assertCountEqual(
[], db_entity.get_field_as_list("supervision_sentences"))
def assert_people_match(self,
expected_people: List[StatePerson],
matched_people: List[schema.StatePerson],
debug: bool = False):
converted_matched = converter.convert_schema_objects_to_entity(
matched_people)
db_expected_with_backedges = converter.convert_entity_people_to_schema_people(
expected_people)
expected_with_backedges = converter.convert_schema_objects_to_entity(
db_expected_with_backedges)
if debug:
print_visible_header_label('EXPECTED')
print_entity_trees(expected_with_backedges)
print_visible_header_label('FINAL')
print_entity_trees(converted_matched)
self.assertEqual(expected_with_backedges, converted_matched)
def assert_people_match(self,
expected_people: List[StatePerson],
matched_people: List[schema.StatePerson],
debug: bool = True):
converted_matched = \
converter.convert_schema_objects_to_entity(matched_people)
db_expected_with_backedges = \
converter.convert_entity_people_to_schema_people(expected_people)
expected_with_backedges = \
converter.convert_schema_objects_to_entity(
db_expected_with_backedges)
if debug:
print('============== EXPECTED WITH BACKEDGES ==============')
print_entity_trees(expected_with_backedges)
print('============== CONVERTED MATCHED ==============')
print_entity_trees(converted_matched)
self.assertCountEqual(expected_with_backedges, converted_matched)
def _assert_people_match(
self, expected_people, matched_people, debug=False):
converted_matched = \
converter.convert_schema_objects_to_entity(matched_people)
db_expected_with_backedges = \
converter.convert_entity_people_to_schema_people(expected_people)
expected_with_backedges = \
converter.convert_schema_objects_to_entity(
db_expected_with_backedges)
clear_db_ids(converted_matched)
clear_db_ids(expected_with_backedges)
if debug:
print('============== EXPECTED WITH BACKEDGES ==============')
print_entity_trees(expected_with_backedges)
print('============== CONVERTED MATCHED ==============')
print_entity_trees(converted_matched)
self.assertCountEqual(expected_with_backedges, converted_matched)
def read_bookings(session):
"""
Reads all bookings in the db.
Args:
session: The transaction to read from
Return:
List of all bookings
"""
return converter.convert_schema_objects_to_entity(
session.query(Booking).all())
def test_getFieldAsList(self):
fine = entities.StateFine.new_with_defaults(external_id='ex1')
fine_2 = entities.StateFine.new_with_defaults(external_id='ex2')
entity = entities.StateSentenceGroup.new_with_defaults(
state_code='us_nc', fines=[fine, fine_2])
db_entity = converter.convert_entity_to_schema_object(entity)
self.assertCountEqual(['us_nc'],
entity.get_field_as_list('state_code'))
self.assertCountEqual([fine, fine_2],
entity.get_field_as_list('fines'))
self.assertCountEqual([],
entity.get_field_as_list('supervision_sentences'))
self.assertCountEqual(['us_nc'],
db_entity.get_field_as_list('state_code'))
self.assertCountEqual([fine, fine_2],
converter.convert_schema_objects_to_entity(
db_entity.get_field_as_list('fines'),
populate_back_edges=False))
self.assertCountEqual([], db_entity.get_field_as_list(
'supervision_sentences'))
def test_setFieldFromList(self):
entity = entities.StateSentenceGroup.new_with_defaults()
fine = entities.StateFine.new_with_defaults(external_id='ex1')
fine_2 = entities.StateFine.new_with_defaults(external_id='ex2')
db_entity = converter.convert_entity_to_schema_object(entity)
db_fine = converter.convert_entity_to_schema_object(fine)
db_fine_2 = converter.convert_entity_to_schema_object(fine_2)
entity.set_field_from_list('state_code', ['us_nc'])
self.assertEqual('us_nc', entity.state_code)
db_entity.set_field_from_list('state_code', ['us_nc'])
self.assertEqual('us_nc', db_entity.state_code)
entity.set_field_from_list('fines', [fine, fine_2])
self.assertCountEqual([fine, fine_2], entity.fines)
db_entity.set_field_from_list('fines', [db_fine, db_fine_2])
self.assertCountEqual([fine, fine_2],
converter.convert_schema_objects_to_entity(
db_entity.fines,
populate_back_edges=False))
def test_setFieldFromList(self) -> None:
entity = entities.StateSentenceGroup.new_with_defaults(
state_code="US_XX",
status=StateSentenceStatus.PRESENT_WITHOUT_INFO)
fine = entities.StateFine.new_with_defaults(
state_code="US_XX",
external_id="ex1",
status=StateFineStatus.PRESENT_WITHOUT_INFO,
)
fine_2 = entities.StateFine.new_with_defaults(
state_code="US_XX",
external_id="ex2",
status=StateFineStatus.PRESENT_WITHOUT_INFO,
)
db_entity = converter.convert_entity_to_schema_object(entity)
if not isinstance(db_entity, schema.StateSentenceGroup):
self.fail(f"Unexpected type for db_entity: {[db_entity]}.")
db_fine = converter.convert_entity_to_schema_object(fine)
db_fine_2 = converter.convert_entity_to_schema_object(fine_2)
entity.set_field_from_list("state_code", ["us_nc"])
self.assertEqual("us_nc", entity.state_code)
db_entity.set_field_from_list("state_code", ["us_nc"])
self.assertEqual("us_nc", db_entity.state_code)
entity.set_field_from_list("fines", [fine, fine_2])
self.assertCountEqual([fine, fine_2], entity.fines)
db_entity.set_field_from_list("fines", [db_fine, db_fine_2])
self.assertCountEqual(
[fine, fine_2],
converter.convert_schema_objects_to_entity(
db_entity.fines, populate_back_edges=False),
)
def assert_schema_object_lists_equal(self, expected: List[StateBase],
actual: List[StateBase]):
self.assertCountEqual(
converter.convert_schema_objects_to_entity(expected),
converter.convert_schema_objects_to_entity(actual),
)
def to_entities(self, schema_objects):
return converter.convert_schema_objects_to_entity(
schema_objects, populate_back_edges=False)
def test_state_threeSentenceGroups_dontPersistAboveThreshold(self):
# Arrange
ingest_info = IngestInfo()
ingest_info.state_people.add(
state_person_id='1_GENERATE',
state_sentence_group_ids=[SENTENCE_GROUP_ID, SENTENCE_GROUP_ID_2])
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID, county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_2,
county_code=COUNTY_CODE)
db_person = schema.StatePerson(person_id=ID, full_name=FULL_NAME_1)
db_sentence_group = schema.StateSentenceGroup(
sentence_group_id=ID,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID,
state_code=REGION_CODE)
db_sentence_group_2 = schema.StateSentenceGroup(
sentence_group_id=ID_2,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_2,
state_code=REGION_CODE)
db_external_id = schema.StatePersonExternalId(person_external_id_id=ID,
state_code=REGION_CODE,
external_id=EXTERNAL_ID,
id_type=ID_TYPE)
db_person.sentence_groups = [db_sentence_group, db_sentence_group_2]
db_person.external_ids = [db_external_id]
db_person_2 = schema.StatePerson(person_id=ID_2, full_name=FULL_NAME_1)
db_sentence_group_2_dup = schema.StateSentenceGroup(
sentence_group_id=ID_3,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_2,
state_code=REGION_CODE)
db_external_id_2 = schema.StatePersonExternalId(
person_external_id_id=ID_2,
state_code=REGION_CODE,
external_id=EXTERNAL_ID_2,
id_type=ID_TYPE)
db_person_2.sentence_groups = [db_sentence_group_2_dup]
db_person_2.external_ids = [db_external_id_2]
# No updates
expected_person = self.to_entity(db_person)
expected_person_2 = self.to_entity(db_person_2)
session = SessionFactory.for_schema_base(StateBase)
session.add(db_person)
session.add(db_person_2)
session.commit()
# Act
persistence.write(ingest_info, DEFAULT_METADATA)
session = SessionFactory.for_schema_base(StateBase)
persons = dao.read_people(session)
# Assert
self.assertEqual([expected_person, expected_person_2],
converter.convert_schema_objects_to_entity(persons))
def test_state_threeSentenceGroups_persistsTwoBelowThreshold(self):
# Arrange
ingest_info = IngestInfo()
ingest_info.state_people.add(state_person_id='1_GENERATE',
state_sentence_group_ids=[
SENTENCE_GROUP_ID,
SENTENCE_GROUP_ID_2,
SENTENCE_GROUP_ID_3
])
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID, county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_2,
county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_3,
county_code=COUNTY_CODE)
db_person = schema.StatePerson(person_id=ID, full_name=FULL_NAME_1)
db_sentence_group = schema.StateSentenceGroup(
sentence_group_id=ID,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID,
state_code=REGION_CODE)
db_sentence_group_2 = schema.StateSentenceGroup(
sentence_group_id=ID_2,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_2,
state_code=REGION_CODE)
db_sentence_group_3 = schema.StateSentenceGroup(
sentence_group_id=ID_3,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_3,
state_code=REGION_CODE)
db_external_id = schema.StatePersonExternalId(person_external_id_id=ID,
state_code=REGION_CODE,
external_id=EXTERNAL_ID,
id_type=ID_TYPE)
db_person.sentence_groups = [
db_sentence_group, db_sentence_group_2, db_sentence_group_3
]
db_person.external_ids = [db_external_id]
db_person_2 = schema.StatePerson(person_id=ID_2, full_name=FULL_NAME_1)
db_sentence_group_3_dup = schema.StateSentenceGroup(
sentence_group_id=ID_4,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_3,
state_code=REGION_CODE)
db_external_id_2 = schema.StatePersonExternalId(
person_external_id_id=ID_2,
state_code=REGION_CODE,
external_id=EXTERNAL_ID_2,
id_type=ID_TYPE)
db_person_2.sentence_groups = [db_sentence_group_3_dup]
db_person_2.external_ids = [db_external_id_2]
expected_person = StatePerson.new_with_defaults(person_id=ID,
full_name=FULL_NAME_1,
external_ids=[],
sentence_groups=[])
expected_external_id = StatePersonExternalId.new_with_defaults(
person_external_id_id=ID,
state_code=REGION_CODE,
external_id=EXTERNAL_ID,
id_type=ID_TYPE,
person=expected_person)
expected_sentence_group = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID,
state_code=REGION_CODE,
county_code=COUNTY_CODE,
person=expected_person)
expected_sentence_group_2 = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID_2,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID_2,
state_code=REGION_CODE,
county_code=COUNTY_CODE,
person=expected_person)
# No county code because errors during match
expected_sentence_group_3 = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID_3,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID_3,
state_code=REGION_CODE,
person=expected_person)
expected_person.external_ids = [expected_external_id]
expected_person.sentence_groups = [
expected_sentence_group, expected_sentence_group_2,
expected_sentence_group_3
]
expected_person_2 = StatePerson.new_with_defaults(
person_id=ID_2, full_name=FULL_NAME_1)
expected_external_id_2 = StatePersonExternalId.new_with_defaults(
person_external_id_id=ID_2,
state_code=REGION_CODE,
external_id=EXTERNAL_ID_2,
id_type=ID_TYPE,
person=expected_person_2)
# No county code because unmatched
expected_sentence_group_3_dup = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID_4,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID_3,
state_code=REGION_CODE,
person=expected_person_2)
expected_person_2.sentence_groups = [expected_sentence_group_3_dup]
expected_person_2.external_ids = [expected_external_id_2]
session = SessionFactory.for_schema_base(StateBase)
session.add(db_person)
session.add(db_person_2)
session.commit()
# Act
persistence.write(ingest_info, DEFAULT_METADATA)
session = SessionFactory.for_schema_base(StateBase)
persons = dao.read_people(session)
# Assert
self.assertEqual([expected_person, expected_person_2],
converter.convert_schema_objects_to_entity(persons))
def test_matchPeople(self):
# Arrange
schema_booking = schema.Booking(
admission_date=_DATE_2,
booking_id=_BOOKING_ID,
custody_status=CustodyStatus.IN_CUSTODY.value,
last_seen_time=_DATE,
first_seen_time=_DATE,
)
schema_person = schema.Person(
person_id=_PERSON_ID,
full_name=_FULL_NAME,
birthdate=_DATE,
jurisdiction_id=_JURISDICTION_ID,
region=_REGION,
bookings=[schema_booking],
)
schema_booking_external_id = schema.Booking(
admission_date=_DATE_2,
booking_id=_BOOKING_ID_ANOTHER,
release_date=_DATE,
custody_status=CustodyStatus.RELEASED.value,
last_seen_time=_DATE,
first_seen_time=_DATE,
)
schema_person_external_id = schema.Person(
person_id=_PERSON_ID_ANOTHER,
external_id=_EXTERNAL_ID,
full_name=_FULL_NAME,
birthdate=_DATE,
jurisdiction_id=_JURISDICTION_ID,
region=_REGION,
bookings=[schema_booking_external_id],
)
with SessionFactory.using_database(self.database_key,
autocommit=False) as session:
session.add(schema_person)
session.add(schema_person_external_id)
session.commit()
ingested_booking = attr.evolve(
converter.convert_schema_object_to_entity(schema_booking),
booking_id=None,
custody_status=CustodyStatus.RELEASED,
)
ingested_person = attr.evolve(
converter.convert_schema_object_to_entity(schema_person),
person_id=None,
bookings=[ingested_booking],
)
ingested_booking_external_id = attr.evolve(
converter.convert_schema_object_to_entity(
schema_booking_external_id),
booking_id=None,
facility=_FACILITY,
)
ingested_person_external_id = attr.evolve(
converter.convert_schema_object_to_entity(
schema_person_external_id),
person_id=None,
bookings=[ingested_booking_external_id],
)
# Act
out = entity_matching.match(
session, _REGION, [ingested_person_external_id, ingested_person])
# Assert
expected_booking = attr.evolve(ingested_booking,
booking_id=_BOOKING_ID)
expected_person = attr.evolve(ingested_person,
person_id=_PERSON_ID,
bookings=[expected_booking])
expected_booking_external_id = attr.evolve(
ingested_booking_external_id, booking_id=_BOOKING_ID_ANOTHER)
expected_person_external_id = attr.evolve(
ingested_person_external_id,
person_id=_PERSON_ID_ANOTHER,
bookings=[expected_booking_external_id],
)
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.people),
[expected_person_external_id, expected_person],
)
self.assertCountEqual(
converter.convert_schema_objects_to_entity(out.orphaned_entities),
[])
self.assertEqual(out.error_count, 0)
def test_state_threeSentenceGroups_persistsTwoBelowThreshold(
self, mock_get_matcher):
"""Ensure that the number of errors is below the ND specific threshold"""
mock_get_matcher.return_value = _PatchedStateEntityMatcher(
region_code=STATE_CODE,
erroring_class=schema.StateSentenceGroup,
erroring_external_ids=[SENTENCE_GROUP_ID],
)
# Set the ENTITY_MATCHING_THRESHOLD to 0, such that we can verify that the forty percent threshold for
# ENTITY_MATCHING_THRESHOLD is dictated by the state-specific override in
# STATE_CODE_TO_ENTITY_MATCHING_THRESHOLD_FORTY_PERCENT.
STATE_ERROR_THRESHOLDS_WITH_FORTY_PERCENT_RATIOS[
ENTITY_MATCHING_THRESHOLD] = 0
# Arrange
ingest_info = IngestInfo()
ingest_info.state_people.add(
state_person_id="1_GENERATE",
state_sentence_group_ids=[SENTENCE_GROUP_ID, SENTENCE_GROUP_ID_2],
)
ingest_info.state_people.add(
state_person_id="2_GENERATE",
state_sentence_group_ids=[SENTENCE_GROUP_ID_3])
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID, county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_2,
county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_3,
county_code=COUNTY_CODE)
db_person = schema.StatePerson(person_id=ID,
full_name=FULL_NAME_1,
state_code=STATE_CODE)
db_sentence_group = schema.StateSentenceGroup(
sentence_group_id=ID,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID,
state_code=STATE_CODE,
)
db_sentence_group_2 = schema.StateSentenceGroup(
sentence_group_id=ID_2,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_2,
state_code=STATE_CODE,
)
db_external_id = schema.StatePersonExternalId(
person_external_id_id=ID,
state_code=STATE_CODE,
external_id=EXTERNAL_ID,
id_type=ID_TYPE,
)
db_person.sentence_groups = [db_sentence_group, db_sentence_group_2]
db_person.external_ids = [db_external_id]
db_person_2 = schema.StatePerson(person_id=ID_2,
full_name=FULL_NAME_1,
state_code=STATE_CODE)
db_sentence_group_3 = schema.StateSentenceGroup(
sentence_group_id=ID_3,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_3,
state_code=STATE_CODE,
)
db_external_id_2 = schema.StatePersonExternalId(
person_external_id_id=ID_2,
state_code=STATE_CODE,
external_id=EXTERNAL_ID_2,
id_type=ID_TYPE,
)
db_person_2.external_ids = [db_external_id_2]
db_person_2.sentence_groups = [db_sentence_group_3]
session = SessionFactory.for_schema_base(StateBase)
session.add(db_person)
session.add(db_person_2)
session.commit()
expected_person = StatePerson.new_with_defaults(
person_id=ID,
full_name=FULL_NAME_1,
external_ids=[],
sentence_groups=[],
state_code=STATE_CODE,
)
expected_external_id = StatePersonExternalId.new_with_defaults(
person_external_id_id=ID,
state_code=STATE_CODE,
external_id=EXTERNAL_ID,
id_type=ID_TYPE,
person=expected_person,
)
# No county code because errors during match
expected_sentence_group = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID,
state_code=STATE_CODE,
person=expected_person,
)
expected_sentence_group_2 = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID_2,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID_2,
state_code=STATE_CODE,
county_code=COUNTY_CODE,
person=expected_person,
)
expected_person.external_ids = [expected_external_id]
expected_person.sentence_groups = [
expected_sentence_group,
expected_sentence_group_2,
]
expected_person_2 = StatePerson.new_with_defaults(
person_id=ID_2, full_name=FULL_NAME_1, state_code=STATE_CODE)
expected_external_id_2 = StatePersonExternalId.new_with_defaults(
person_external_id_id=ID_2,
state_code=STATE_CODE,
external_id=EXTERNAL_ID_2,
id_type=ID_TYPE,
person=expected_person_2,
)
expected_sentence_group_3 = StateSentenceGroup.new_with_defaults(
sentence_group_id=ID_3,
status=StateSentenceStatus.EXTERNAL_UNKNOWN,
external_id=SENTENCE_GROUP_ID_3,
state_code=STATE_CODE,
county_code=COUNTY_CODE,
person=expected_person_2,
)
expected_person_2.sentence_groups = [expected_sentence_group_3]
expected_person_2.external_ids = [expected_external_id_2]
# Act
persistence.write(ingest_info, DEFAULT_METADATA)
session = SessionFactory.for_schema_base(StateBase)
persons = dao.read_people(session)
# Assert
self.assertEqual(
[expected_person, expected_person_2],
converter.convert_schema_objects_to_entity(persons),
)
def test_state_threeSentenceGroups_dontPersistAboveThreshold(
self, mock_get_matcher):
# Arrange
mock_get_matcher.return_value = _PatchedStateEntityMatcher(
region_code=STATE_CODE,
erroring_class=schema.StateSentenceGroup,
erroring_external_ids=[SENTENCE_GROUP_ID, SENTENCE_GROUP_ID_4],
)
# Arrange
ingest_info = IngestInfo()
ingest_info.state_people.add(
state_person_id="1_GENERATE",
state_sentence_group_ids=[SENTENCE_GROUP_ID, SENTENCE_GROUP_ID_2],
)
ingest_info.state_people.add(
state_person_id="2_GENERATE",
state_sentence_group_ids=[
SENTENCE_GROUP_ID_3, SENTENCE_GROUP_ID_4
],
)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID, county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_2,
county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_3,
county_code=COUNTY_CODE)
ingest_info.state_sentence_groups.add(
state_sentence_group_id=SENTENCE_GROUP_ID_4,
county_code=COUNTY_CODE)
db_person = schema.StatePerson(person_id=ID,
full_name=FULL_NAME_1,
state_code=STATE_CODE)
db_sentence_group = schema.StateSentenceGroup(
sentence_group_id=ID,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID,
state_code=STATE_CODE,
)
db_sentence_group_2 = schema.StateSentenceGroup(
sentence_group_id=ID_2,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_2,
state_code=STATE_CODE,
)
db_external_id = schema.StatePersonExternalId(
person_external_id_id=ID,
state_code=STATE_CODE,
external_id=EXTERNAL_ID,
id_type=ID_TYPE,
)
db_person.sentence_groups = [db_sentence_group, db_sentence_group_2]
db_person.external_ids = [db_external_id]
db_person_2 = schema.StatePerson(person_id=ID_2,
full_name=FULL_NAME_1,
state_code=STATE_CODE)
db_sentence_group_3 = schema.StateSentenceGroup(
sentence_group_id=ID_3,
status=StateSentenceStatus.EXTERNAL_UNKNOWN.value,
external_id=SENTENCE_GROUP_ID_3,
state_code=STATE_CODE,
)
db_external_id_2 = schema.StatePersonExternalId(
person_external_id_id=ID_2,
state_code=STATE_CODE,
external_id=EXTERNAL_ID_2,
id_type=ID_TYPE,
)
db_person_2.external_ids = [db_external_id_2]
db_person_2.sentence_groups = [db_sentence_group_3]
# No updates
expected_person = self.to_entity(db_person)
expected_person_2 = self.to_entity(db_person_2)
session = SessionFactory.for_schema_base(StateBase)
session.add(db_person)
session.add(db_person_2)
session.commit()
# Act
persistence.write(ingest_info, DEFAULT_METADATA)
session = SessionFactory.for_schema_base(StateBase)
persons = dao.read_people(session)
# Assert
self.assertEqual(
[expected_person, expected_person_2],
converter.convert_schema_objects_to_entity(persons),
)
