def testParseSentence_completionDateWithoutStatus_marksAsCompleted(self): # Arrange ingest_sentence = ingest_info_pb2.Sentence( sentence_id="SENTENCE_ID", completion_date="2018-1-1", ) # Act sentence_builder = entities.Sentence.builder() sentence.copy_fields_to_builder( sentence_builder, ingest_sentence, FakeIngestMetadata.for_county( region="REGION", ingest_time=datetime.datetime(year=2018, month=6, day=8), ), ) result = sentence_builder.build() # Assert expected_result = entities.Sentence.new_with_defaults( external_id="SENTENCE_ID", completion_date=datetime.date(year=2018, month=1, day=1), status=SentenceStatus.COMPLETED, ) self.assertEqual(result, expected_result)
def testConvert_PersonInferredBooking(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID", ingest_time=_INGEST_TIME) ingest_info = IngestInfo() ingest_info.people.add() # Act result = self._convert_and_throw_on_errors(ingest_info, metadata) # Assert expected_result = [ Person.new_with_defaults( region="REGION", jurisdiction_id="JURISDICTION_ID", bookings=[ Booking.new_with_defaults( admission_date_inferred=True, first_seen_time=_INGEST_TIME, last_seen_time=_INGEST_TIME, admission_date=_INGEST_TIME.date(), custody_status=CustodyStatus.PRESENT_WITHOUT_INFO, ) ], ) ] self.assertEqual(result, expected_result)
def testParseStatePerson_WithSurnameAndGivenNames_UsesFullNameAsJson(self): # Arrange metadata = FakeIngestMetadata.for_state(region="us_xx") ingest_person = ingest_info_pb2.StatePerson( state_code="us_xx", surname='UNESCAPED,SURNAME"WITH-CHARS"', given_names="GIVEN_NAMES", middle_names="MIDDLE_NAMES", ) # Act state_person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build(StatePersonFactory.deserialize) # Assert expected_full_name = ( '{{"given_names": "{}", "middle_names": "{}", "surname": "{}"}}'.format( "GIVEN_NAMES", "MIDDLE_NAMES", 'UNESCAPED,SURNAME\\"WITH-CHARS\\"' ) ) expected_result = entities.StatePerson.new_with_defaults( state_code="US_XX", full_name=expected_full_name ) self.assertEqual(result, expected_result)
def testParseBooking_releaseDateInFuture_setAsProjected(self): # Arrange metadata = FakeIngestMetadata.for_county(region="REGION", ingest_time=_INGEST_TIME) ingest_booking = ingest_info_pb2.Booking( booking_id="BOOKING_ID", admission_date="2/3/1000", release_date="1/2/2020", ) # Act booking.copy_fields_to_builder(self.subject, ingest_booking, metadata) result = self.subject.build() # Assert expected_result = entities.Booking.new_with_defaults( external_id="BOOKING_ID", admission_date=date(year=1000, month=2, day=3), admission_date_inferred=False, release_date=None, release_date_inferred=None, projected_release_date=date(year=2020, month=1, day=2), custody_status=CustodyStatus.PRESENT_WITHOUT_INFO, last_seen_time=_INGEST_TIME, first_seen_time=_INGEST_TIME, ) self.assertEqual(result, expected_result)
def testParseBooking_facilityId(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", ingest_time=_INGEST_TIME, ) ingest_booking = ingest_info_pb2.Booking( facility_id="1234567890123456") # Act booking.copy_fields_to_builder(self.subject, ingest_booking, metadata) result = self.subject.build() # Assert expected_result = entities.Booking.new_with_defaults( facility_id="1234567890123456", admission_date=_INGEST_TIME.date(), admission_date_inferred=True, last_seen_time=_INGEST_TIME, first_seen_time=_INGEST_TIME, release_date=None, release_date_inferred=None, custody_status=CustodyStatus.PRESENT_WITHOUT_INFO, ) self.assertEqual(result, expected_result)
def testParseBooking_releaseDateWithoutStatus_marksAsReleased(self): # Arrange metadata = FakeIngestMetadata.for_county(region="REGION", ingest_time=_INGEST_TIME) ingest_booking = ingest_info_pb2.Booking( booking_id="BOOKING_ID", release_date="2/1/2019", ) # Act booking.copy_fields_to_builder(self.subject, ingest_booking, metadata) result = self.subject.build() # Assert expected_result = entities.Booking.new_with_defaults( external_id="BOOKING_ID", admission_date=_INGEST_TIME.date(), admission_date_inferred=True, release_date=date(year=2019, month=2, day=1), release_date_inferred=False, custody_status=CustodyStatus.RELEASED, last_seen_time=_INGEST_TIME, first_seen_time=_INGEST_TIME, ) self.assertEqual(result, expected_result)
def testParseStateBond(self) -> None: # Arrange ingest_bond = ingest_info_pb2.StateBond( status="ACTIVE", bond_type="CASH", state_bond_id="BOND_ID", date_paid="1/2/2111", state_code="us_nd", county_code="BISMARCK", bond_agent="AGENT", amount="$125.00", ) # Act result = state_bond.convert(ingest_bond, FakeIngestMetadata.for_state("us_nd")) # Assert expected_result = entities.StateBond( status=BondStatus.SET, status_raw_text="ACTIVE", bond_type=BondType.CASH, bond_type_raw_text="CASH", external_id="BOND_ID", date_paid=date(year=2111, month=1, day=2), state_code="US_ND", county_code="BISMARCK", bond_agent="AGENT", amount_dollars=125, ) self.assertEqual(result, expected_result)
def testCompletionDateInFuture(self): # Arrange ingest_sentence = ingest_info_pb2.Sentence( sentence_id="SENTENCE_ID", completion_date="2020-1-1", ) # Act sentence_builder = entities.Sentence.builder() sentence.copy_fields_to_builder( sentence_builder, ingest_sentence, FakeIngestMetadata.for_county( region="REGION", ingest_time=datetime.datetime(year=2018, month=6, day=8), ), ) result = sentence_builder.build() # Assert expected_result = entities.Sentence.new_with_defaults( external_id="SENTENCE_ID", completion_date=None, projected_completion_date=datetime.date(year=2020, month=1, day=1), status=SentenceStatus.PRESENT_WITHOUT_INFO, ) self.assertEqual(result, expected_result)
def testParsesPerson(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID") ingest_person = ingest_info_pb2.Person( full_name="FULL_NAME", birthdate="12-31-1999", gender="MALE", race="WHITE", ethnicity="HISPANIC", place_of_residence="NNN\n STREET \t ZIP", ) # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( full_name='{{"full_name": "{}"}}'.format("FULL_NAME"), birthdate=date(year=1999, month=12, day=31), birthdate_inferred_from_age=False, gender=Gender.MALE, gender_raw_text="MALE", race=Race.WHITE, race_raw_text="WHITE", ethnicity=Ethnicity.HISPANIC, ethnicity_raw_text="HISPANIC", residency_status=ResidencyStatus.PERMANENT, region="REGION", jurisdiction_id="JURISDICTION_ID", ) self.assertEqual(result, expected_result)
def testParsePerson_WithSurnameAndGivenNames_UsesFullNameAsJson(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID") ingest_person = ingest_info_pb2.Person( surname='UNESCAPED,SURNAME"WITH-CHARS"', given_names="GIVEN_NAMES", middle_names="MIDDLE_NAMES", ) # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_full_name = ( '{{"given_names": "{}", "middle_names": "{}", "surname": "{}"}}'. format("GIVEN_NAMES", "MIDDLE_NAMES", 'UNESCAPED,SURNAME\\"WITH-CHARS\\"')) expected_result = entities.Person.new_with_defaults( region="REGION", jurisdiction_id="JURISDICTION_ID", full_name=expected_full_name, ) self.assertEqual(result, expected_result)
def testParseCharge_MapAcrossFields(self): # Arrange overrides_builder = EnumOverrides.Builder() overrides_builder.add("FELONY", ChargeClass.FELONY, ChargeDegree) overrides_builder.add("FIRST DEGREE", ChargeDegree.FIRST, ChargeClass) metadata = FakeIngestMetadata.for_county( region="REGION", enum_overrides=overrides_builder.build() ) ingest_charge = ingest_info_pb2.Charge( charge_class="first degree", degree="felony" ) # Act charge.copy_fields_to_builder(self.subject, ingest_charge, metadata) result = self.subject.build() # Assert expected_result = entities.Charge.new_with_defaults( degree=ChargeDegree.FIRST, degree_raw_text="FELONY", charge_class=ChargeClass.FELONY, class_raw_text="FIRST DEGREE", status=ChargeStatus.PRESENT_WITHOUT_INFO, ) self.assertEqual(result, expected_result)
def testParsesStatePerson(self): # Arrange metadata = FakeIngestMetadata.for_state(region="us_nd") ingest_person = ingest_info_pb2.StatePerson( gender="MALE", full_name="FULL_NAME", birthdate="12-31-1999", current_address="NNN\n STREET \t ZIP", ) # Act state_person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build(StatePersonFactory.deserialize) # Assert expected_result = entities.StatePerson.new_with_defaults( gender=Gender.MALE, gender_raw_text="MALE", full_name='{"full_name": "FULL_NAME"}', birthdate=date(year=1999, month=12, day=31), birthdate_inferred_from_age=False, current_address="NNN STREET ZIP", residency_status=ResidencyStatus.PERMANENT, state_code="US_ND", ) self.assertEqual(result, expected_result)
def testParsePerson_RaceIsEthnicity(self): class ScraperWithDefaultOverrides(BaseScraper): """Class created so BaseScraper's enum_overrides can be used.""" def __init__(self): # pylint: disable=super-init-not-called self.region = Mock() def populate_data(self, _, __, ___): pass # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID", enum_overrides=ScraperWithDefaultOverrides().get_enum_overrides(), ) ingest_person = ingest_info_pb2.Person(race="HISPANIC") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( region="REGION", jurisdiction_id="JURISDICTION_ID", ethnicity=Ethnicity.HISPANIC, race_raw_text="HISPANIC", ) self.assertEqual(result, expected_result)
def testParseBooking_MapAcrossFields(self): # Arrange overrides_builder = EnumOverrides.Builder() overrides_builder.add("WORK RELEASE", Classification.WORK_RELEASE, AdmissionReason) overrides_builder.add("transfer", AdmissionReason.TRANSFER, CustodyStatus) metadata = FakeIngestMetadata.for_county( region="REGION", ingest_time=_INGEST_TIME, enum_overrides=overrides_builder.build(), ) ingest_booking = ingest_info_pb2.Booking( admission_reason="work release", custody_status="transfer") # Act booking.copy_fields_to_builder(self.subject, ingest_booking, metadata) result = self.subject.build() # Assert expected_result = entities.Booking.new_with_defaults( admission_date=_INGEST_TIME.date(), admission_reason=AdmissionReason.TRANSFER, admission_reason_raw_text="WORK RELEASE", admission_date_inferred=True, custody_status=CustodyStatus.PRESENT_WITHOUT_INFO, custody_status_raw_text="TRANSFER", classification=Classification.WORK_RELEASE, last_seen_time=_INGEST_TIME, first_seen_time=_INGEST_TIME, ) self.assertEqual(result, expected_result)
def testParseSentence(self): # Arrange ingest_sentence = ingest_info_pb2.Sentence( sentence_id="SENTENCE_ID", min_length="1", date_imposed="2000-1-1", post_release_supervision_length="", ) # Act sentence_builder = entities.Sentence.builder() sentence.copy_fields_to_builder( sentence_builder, ingest_sentence, FakeIngestMetadata.for_county( region="REGION", ingest_time=datetime.datetime(year=2018, month=6, day=8), ), ) result = sentence_builder.build() # Assert expected_result = entities.Sentence.new_with_defaults( external_id="SENTENCE_ID", min_length_days=1, post_release_supervision_length_days=0, date_imposed=datetime.date(year=2000, month=1, day=1), status=SentenceStatus.PRESENT_WITHOUT_INFO, ) self.assertEqual(result, expected_result)
def testConvert_CannotConvertField_RaisesValueError(self): # Arrange metadata = metadata = FakeIngestMetadata.for_state(region="us_xx") ingest_info = IngestInfo() ingest_info.state_people.add(birthdate="NOT_A_DATE") # Act + Assert with self.assertRaises(ValueError): self._convert_and_throw_on_errors(ingest_info, metadata)
def testParseStatePerson_WithSurnameAndFullname_ThrowsException(self): # Arrange metadata = FakeIngestMetadata.for_state(region="us_xx") ingest_person = ingest_info_pb2.StatePerson( full_name="LAST,FIRST", surname="LAST" ) # Arrange + Act with self.assertRaises(ValueError): state_person.copy_fields_to_builder(self.subject, ingest_person, metadata)
def testParseBooking_facilityId_doesNotExist(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", ingest_time=_INGEST_TIME, ) ingest_booking = ingest_info_pb2.Booking( facility_id="1234567890123456" # invalid FID ) with self.assertRaises(ValueError): booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
def testConvert_TotalBondWithCharge_SetsTotalBondOnCharge(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID", ingest_time=_INGEST_TIME) ingest_info = IngestInfo() ingest_info.people.add(booking_ids=["BOOKING_ID"]) ingest_info.bookings.add(booking_id="BOOKING_ID", total_bond_amount="$100", charge_ids=["CHARGE_ID"]) ingest_info.charges.add(charge_id="CHARGE_ID") # Act result = self._convert_and_throw_on_errors(ingest_info, metadata) # Assert expected_result = [ Person.new_with_defaults( region="REGION", jurisdiction_id="JURISDICTION_ID", bookings=[ Booking.new_with_defaults( external_id="BOOKING_ID", admission_date=_INGEST_TIME.date(), admission_date_inferred=True, first_seen_time=_INGEST_TIME, last_seen_time=_INGEST_TIME, custody_status=CustodyStatus.PRESENT_WITHOUT_INFO, charges=[ Charge.new_with_defaults( external_id="CHARGE_ID_COUNT_1", status=ChargeStatus.PRESENT_WITHOUT_INFO, bond=Bond.new_with_defaults( amount_dollars=100, status=BondStatus.PRESENT_WITHOUT_INFO, bond_type=BondType.CASH, ), ) ], ) ], ) ] self.assertEqual(result, expected_result)
def testConvert_MultipleOpenBookings_RaisesValueError(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID", ingest_time=_INGEST_TIME) ingest_info = IngestInfo() ingest_info.people.add(booking_ids=["BOOKING_ID1", "BOOKING_ID2"]) ingest_info.bookings.add(booking_id="BOOKING_ID1", admission_date="3/14/2020") ingest_info.bookings.add(booking_id="BOOKING_ID2", admission_date="3/16/2020") # Act + Assert with self.assertRaises(ValueError): self._convert_and_throw_on_errors(ingest_info, metadata)
def testParsePerson_NoNames_FullNameIsNone(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID") ingest_person = ingest_info_pb2.Person(person_id="1234") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( region="REGION", jurisdiction_id="JURISDICTION_ID", external_id="1234") self.assertEqual(result, expected_result)
def testParseStatePerson_TakesLastZipCodeMatch(self): # Arrange metadata = FakeIngestMetadata.for_state(region="us_nd") # 5-digit address could be mistaken for a zip code ingest_person = ingest_info_pb2.StatePerson(current_address="12345 Main 58503") # Act state_person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build(StatePersonFactory.deserialize) # Assert expected_result = entities.StatePerson.new_with_defaults( current_address="12345 MAIN 58503", residency_status=ResidencyStatus.PERMANENT, state_code="US_ND", ) self.assertEqual(result, expected_result)
def testParseStatePerson_NoiseInPlaceOfResidence_ParsesResidency(self): # Arrange metadata = FakeIngestMetadata.for_state(region="us_xx") ingest_person = ingest_info_pb2.StatePerson( current_address="transient moves around" ) # Act state_person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build(StatePersonFactory.deserialize) # Assert expected_result = entities.StatePerson.new_with_defaults( current_address="TRANSIENT MOVES AROUND", residency_status=ResidencyStatus.TRANSIENT, state_code="US_XX", ) self.assertEqual(result, expected_result)
def testParsePerson_NoiseInPlaceOfResidence_ParsesResidencyStatus(self): # Arrange metadata = FakeIngestMetadata.for_county( region="us_ky_allen", jurisdiction_id="JURISDICTION_ID") ingest_person = ingest_info_pb2.Person( place_of_residence="transient moves around") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( residency_status=ResidencyStatus.TRANSIENT, region="us_ky_allen", jurisdiction_id="JURISDICTION_ID", ) self.assertEqual(result, expected_result)
def testParsePerson_ResidenceAndStatusCombined(self): # Arrange metadata = FakeIngestMetadata.for_county( region="us_ky_allen", jurisdiction_id="JURISDICTION_ID") ingest_person = ingest_info_pb2.Person( place_of_residence="42164 homeless") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( resident_of_region=True, residency_status=ResidencyStatus.HOMELESS, region="us_ky_allen", jurisdiction_id="JURISDICTION_ID", ) self.assertEqual(result, expected_result)
def testParsePerson_InfersBirthdateFromAge(self, mock_datetime): # Arrange mock_datetime.now.return_value = _NOW metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID") ingest_person = ingest_info_pb2.Person(age="27") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( region="REGION", jurisdiction_id="JURISDICTION_ID", birthdate=datetime(year=_NOW.year - 27, month=1, day=1).date(), birthdate_inferred_from_age=True, ) self.assertEqual(result, expected_result)
def testParseBooking(self): # Arrange metadata = FakeIngestMetadata.for_county(region="REGION", ingest_time=_INGEST_TIME) ingest_booking = ingest_info_pb2.Booking( booking_id="BOOKING_ID", admission_date="2/3/1000", admission_reason="New Commitment", release_date="1/2/2017", projected_release_date="5/20/2020", release_reason="Transfer", custody_status="Held Elsewhere", classification="Low", ) # Act booking.copy_fields_to_builder(self.subject, ingest_booking, metadata) result = self.subject.build() # Assert expected_result = entities.Booking.new_with_defaults( external_id="BOOKING_ID", admission_date=date(year=1000, month=2, day=3), admission_reason=AdmissionReason.NEW_COMMITMENT, admission_reason_raw_text="NEW COMMITMENT", admission_date_inferred=False, release_date=date(year=2017, month=1, day=2), release_date_inferred=False, projected_release_date=date(year=2020, month=5, day=20), release_reason=ReleaseReason.TRANSFER, release_reason_raw_text="TRANSFER", custody_status=CustodyStatus.HELD_ELSEWHERE, custody_status_raw_text="HELD ELSEWHERE", classification=Classification.LOW, classification_raw_text="LOW", last_seen_time=_INGEST_TIME, first_seen_time=_INGEST_TIME, ) self.assertEqual(result, expected_result)
def testParsePerson_NotResidentOfState(self): # Arrange metadata = FakeIngestMetadata.for_county( region="us_ky", jurisdiction_id="JURISDICTION_ID") # 10011 is in New York ingest_person = ingest_info_pb2.Person( place_of_residence="123 Main 10011") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( resident_of_region=False, residency_status=ResidencyStatus.PERMANENT, region="us_ky", jurisdiction_id="JURISDICTION_ID", ) self.assertEqual(result, expected_result)
def testParsePerson_TakesLastZipCodeMatch(self): # Arrange metadata = FakeIngestMetadata.for_county( region="us_ky_allen", jurisdiction_id="JURISDICTION_ID") # 5-digit address could be mistaken for a zip code ingest_person = ingest_info_pb2.Person( place_of_residence="12345 Main 42164") # Act person.copy_fields_to_builder(self.subject, ingest_person, metadata) result = self.subject.build() # Assert expected_result = entities.Person.new_with_defaults( resident_of_region=True, residency_status=ResidencyStatus.PERMANENT, region="us_ky_allen", jurisdiction_id="JURISDICTION_ID", ) self.assertEqual(result, expected_result)
def testConvert_TotalBondWithMultipleBonds_ThrowsException(self): # Arrange metadata = FakeIngestMetadata.for_county( region="REGION", jurisdiction_id="JURISDICTION_ID", ingest_time=_INGEST_TIME) ingest_info = IngestInfo() ingest_info.people.add(booking_ids=["BOOKING_ID"]) ingest_info.bookings.add( booking_id="BOOKING_ID", total_bond_amount="$100", charge_ids=["CHARGE_ID", "CHARGE_ID_2"], ) ingest_info.charges.add(charge_id="CHARGE_ID", bond_id="BOND_ID") ingest_info.charges.add(charge_id="CHARGE_ID_2", bond_id="BOND_ID_2") ingest_info.bonds.add(bond_id="BOND_ID") ingest_info.bonds.add(bond_id="BOND_ID_2") # Act + Assert with self.assertRaises(ValueError): self._convert_and_throw_on_errors(ingest_info, metadata)