def testParseSentence_completionDateWithoutStatus_marksAsCompleted(self):
# Arrange
ingest_sentence = ingest_info_pb2.Sentence(
sentence_id="SENTENCE_ID",
completion_date="2018-1-1",
)
# Act
sentence_builder = entities.Sentence.builder()
sentence.copy_fields_to_builder(
sentence_builder,
ingest_sentence,
FakeIngestMetadata.for_county(
region="REGION",
ingest_time=datetime.datetime(year=2018, month=6, day=8),
),
)
result = sentence_builder.build()
# Assert
expected_result = entities.Sentence.new_with_defaults(
external_id="SENTENCE_ID",
completion_date=datetime.date(year=2018, month=1, day=1),
status=SentenceStatus.COMPLETED,
)
self.assertEqual(result, expected_result)
def testConvert_PersonInferredBooking(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add()
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
bookings=[
Booking.new_with_defaults(
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
admission_date=_INGEST_TIME.date(),
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
)
],
)
]
self.assertEqual(result, expected_result)
def testParseStatePerson_WithSurnameAndGivenNames_UsesFullNameAsJson(self):
# Arrange
metadata = FakeIngestMetadata.for_state(region="us_xx")
ingest_person = ingest_info_pb2.StatePerson(
state_code="us_xx",
surname='UNESCAPED,SURNAME"WITH-CHARS"',
given_names="GIVEN_NAMES",
middle_names="MIDDLE_NAMES",
)
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build(StatePersonFactory.deserialize)
# Assert
expected_full_name = (
'{{"given_names": "{}", "middle_names": "{}", "surname": "{}"}}'.format(
"GIVEN_NAMES", "MIDDLE_NAMES", 'UNESCAPED,SURNAME\\"WITH-CHARS\\"'
)
)
expected_result = entities.StatePerson.new_with_defaults(
state_code="US_XX", full_name=expected_full_name
)
self.assertEqual(result, expected_result)
def testParseBooking_releaseDateInFuture_setAsProjected(self):
# Arrange
metadata = FakeIngestMetadata.for_county(region="REGION",
ingest_time=_INGEST_TIME)
ingest_booking = ingest_info_pb2.Booking(
booking_id="BOOKING_ID",
admission_date="2/3/1000",
release_date="1/2/2020",
)
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=date(year=1000, month=2, day=3),
admission_date_inferred=False,
release_date=None,
release_date_inferred=None,
projected_release_date=date(year=2020, month=1, day=2),
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testParseBooking_facilityId(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
ingest_time=_INGEST_TIME,
)
ingest_booking = ingest_info_pb2.Booking(
facility_id="1234567890123456")
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
facility_id="1234567890123456",
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
release_date=None,
release_date_inferred=None,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
)
self.assertEqual(result, expected_result)
def testParseBooking_releaseDateWithoutStatus_marksAsReleased(self):
# Arrange
metadata = FakeIngestMetadata.for_county(region="REGION",
ingest_time=_INGEST_TIME)
ingest_booking = ingest_info_pb2.Booking(
booking_id="BOOKING_ID",
release_date="2/1/2019",
)
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
release_date=date(year=2019, month=2, day=1),
release_date_inferred=False,
custody_status=CustodyStatus.RELEASED,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testParseStateBond(self) -> None:
# Arrange
ingest_bond = ingest_info_pb2.StateBond(
status="ACTIVE",
bond_type="CASH",
state_bond_id="BOND_ID",
date_paid="1/2/2111",
state_code="us_nd",
county_code="BISMARCK",
bond_agent="AGENT",
amount="$125.00",
)
# Act
result = state_bond.convert(ingest_bond,
FakeIngestMetadata.for_state("us_nd"))
# Assert
expected_result = entities.StateBond(
status=BondStatus.SET,
status_raw_text="ACTIVE",
bond_type=BondType.CASH,
bond_type_raw_text="CASH",
external_id="BOND_ID",
date_paid=date(year=2111, month=1, day=2),
state_code="US_ND",
county_code="BISMARCK",
bond_agent="AGENT",
amount_dollars=125,
)
self.assertEqual(result, expected_result)
def testCompletionDateInFuture(self):
# Arrange
ingest_sentence = ingest_info_pb2.Sentence(
sentence_id="SENTENCE_ID",
completion_date="2020-1-1",
)
# Act
sentence_builder = entities.Sentence.builder()
sentence.copy_fields_to_builder(
sentence_builder,
ingest_sentence,
FakeIngestMetadata.for_county(
region="REGION",
ingest_time=datetime.datetime(year=2018, month=6, day=8),
),
)
result = sentence_builder.build()
# Assert
expected_result = entities.Sentence.new_with_defaults(
external_id="SENTENCE_ID",
completion_date=None,
projected_completion_date=datetime.date(year=2020, month=1, day=1),
status=SentenceStatus.PRESENT_WITHOUT_INFO,
)
self.assertEqual(result, expected_result)
def testParsesPerson(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.Person(
full_name="FULL_NAME",
birthdate="12-31-1999",
gender="MALE",
race="WHITE",
ethnicity="HISPANIC",
place_of_residence="NNN\n STREET \t ZIP",
)
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
full_name='{{"full_name": "{}"}}'.format("FULL_NAME"),
birthdate=date(year=1999, month=12, day=31),
birthdate_inferred_from_age=False,
gender=Gender.MALE,
gender_raw_text="MALE",
race=Race.WHITE,
race_raw_text="WHITE",
ethnicity=Ethnicity.HISPANIC,
ethnicity_raw_text="HISPANIC",
residency_status=ResidencyStatus.PERMANENT,
region="REGION",
jurisdiction_id="JURISDICTION_ID",
)
self.assertEqual(result, expected_result)
def testParsePerson_WithSurnameAndGivenNames_UsesFullNameAsJson(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.Person(
surname='UNESCAPED,SURNAME"WITH-CHARS"',
given_names="GIVEN_NAMES",
middle_names="MIDDLE_NAMES",
)
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_full_name = (
'{{"given_names": "{}", "middle_names": "{}", "surname": "{}"}}'.
format("GIVEN_NAMES", "MIDDLE_NAMES",
'UNESCAPED,SURNAME\\"WITH-CHARS\\"'))
expected_result = entities.Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
full_name=expected_full_name,
)
self.assertEqual(result, expected_result)
def testParseCharge_MapAcrossFields(self):
# Arrange
overrides_builder = EnumOverrides.Builder()
overrides_builder.add("FELONY", ChargeClass.FELONY, ChargeDegree)
overrides_builder.add("FIRST DEGREE", ChargeDegree.FIRST, ChargeClass)
metadata = FakeIngestMetadata.for_county(
region="REGION", enum_overrides=overrides_builder.build()
)
ingest_charge = ingest_info_pb2.Charge(
charge_class="first degree", degree="felony"
)
# Act
charge.copy_fields_to_builder(self.subject, ingest_charge, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Charge.new_with_defaults(
degree=ChargeDegree.FIRST,
degree_raw_text="FELONY",
charge_class=ChargeClass.FELONY,
class_raw_text="FIRST DEGREE",
status=ChargeStatus.PRESENT_WITHOUT_INFO,
)
self.assertEqual(result, expected_result)
def testParsesStatePerson(self):
# Arrange
metadata = FakeIngestMetadata.for_state(region="us_nd")
ingest_person = ingest_info_pb2.StatePerson(
gender="MALE",
full_name="FULL_NAME",
birthdate="12-31-1999",
current_address="NNN\n STREET \t ZIP",
)
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build(StatePersonFactory.deserialize)
# Assert
expected_result = entities.StatePerson.new_with_defaults(
gender=Gender.MALE,
gender_raw_text="MALE",
full_name='{"full_name": "FULL_NAME"}',
birthdate=date(year=1999, month=12, day=31),
birthdate_inferred_from_age=False,
current_address="NNN STREET ZIP",
residency_status=ResidencyStatus.PERMANENT,
state_code="US_ND",
)
self.assertEqual(result, expected_result)
def testParsePerson_RaceIsEthnicity(self):
class ScraperWithDefaultOverrides(BaseScraper):
"""Class created so BaseScraper's enum_overrides can be used."""
def __init__(self): # pylint: disable=super-init-not-called
self.region = Mock()
def populate_data(self, _, __, ___):
pass
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
enum_overrides=ScraperWithDefaultOverrides().get_enum_overrides(),
)
ingest_person = ingest_info_pb2.Person(race="HISPANIC")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
ethnicity=Ethnicity.HISPANIC,
race_raw_text="HISPANIC",
)
self.assertEqual(result, expected_result)
def testParseBooking_MapAcrossFields(self):
# Arrange
overrides_builder = EnumOverrides.Builder()
overrides_builder.add("WORK RELEASE", Classification.WORK_RELEASE,
AdmissionReason)
overrides_builder.add("transfer", AdmissionReason.TRANSFER,
CustodyStatus)
metadata = FakeIngestMetadata.for_county(
region="REGION",
ingest_time=_INGEST_TIME,
enum_overrides=overrides_builder.build(),
)
ingest_booking = ingest_info_pb2.Booking(
admission_reason="work release", custody_status="transfer")
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_reason=AdmissionReason.TRANSFER,
admission_reason_raw_text="WORK RELEASE",
admission_date_inferred=True,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
custody_status_raw_text="TRANSFER",
classification=Classification.WORK_RELEASE,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testParseSentence(self):
# Arrange
ingest_sentence = ingest_info_pb2.Sentence(
sentence_id="SENTENCE_ID",
min_length="1",
date_imposed="2000-1-1",
post_release_supervision_length="",
)
# Act
sentence_builder = entities.Sentence.builder()
sentence.copy_fields_to_builder(
sentence_builder,
ingest_sentence,
FakeIngestMetadata.for_county(
region="REGION",
ingest_time=datetime.datetime(year=2018, month=6, day=8),
),
)
result = sentence_builder.build()
# Assert
expected_result = entities.Sentence.new_with_defaults(
external_id="SENTENCE_ID",
min_length_days=1,
post_release_supervision_length_days=0,
date_imposed=datetime.date(year=2000, month=1, day=1),
status=SentenceStatus.PRESENT_WITHOUT_INFO,
)
self.assertEqual(result, expected_result)
def testConvert_CannotConvertField_RaisesValueError(self):
# Arrange
metadata = metadata = FakeIngestMetadata.for_state(region="us_xx")
ingest_info = IngestInfo()
ingest_info.state_people.add(birthdate="NOT_A_DATE")
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)
def testParseStatePerson_WithSurnameAndFullname_ThrowsException(self):
# Arrange
metadata = FakeIngestMetadata.for_state(region="us_xx")
ingest_person = ingest_info_pb2.StatePerson(
full_name="LAST,FIRST", surname="LAST"
)
# Arrange + Act
with self.assertRaises(ValueError):
state_person.copy_fields_to_builder(self.subject, ingest_person, metadata)
def testParseBooking_facilityId_doesNotExist(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
ingest_time=_INGEST_TIME,
)
ingest_booking = ingest_info_pb2.Booking(
facility_id="1234567890123456" # invalid FID
)
with self.assertRaises(ValueError):
booking.copy_fields_to_builder(self.subject, ingest_booking,
metadata)
def testConvert_TotalBondWithCharge_SetsTotalBondOnCharge(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=["BOOKING_ID"])
ingest_info.bookings.add(booking_id="BOOKING_ID",
total_bond_amount="$100",
charge_ids=["CHARGE_ID"])
ingest_info.charges.add(charge_id="CHARGE_ID")
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
bookings=[
Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
charges=[
Charge.new_with_defaults(
external_id="CHARGE_ID_COUNT_1",
status=ChargeStatus.PRESENT_WITHOUT_INFO,
bond=Bond.new_with_defaults(
amount_dollars=100,
status=BondStatus.PRESENT_WITHOUT_INFO,
bond_type=BondType.CASH,
),
)
],
)
],
)
]
self.assertEqual(result, expected_result)
def testConvert_MultipleOpenBookings_RaisesValueError(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=["BOOKING_ID1", "BOOKING_ID2"])
ingest_info.bookings.add(booking_id="BOOKING_ID1",
admission_date="3/14/2020")
ingest_info.bookings.add(booking_id="BOOKING_ID2",
admission_date="3/16/2020")
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)
def testParsePerson_NoNames_FullNameIsNone(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.Person(person_id="1234")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
external_id="1234")
self.assertEqual(result, expected_result)
def testParseStatePerson_TakesLastZipCodeMatch(self):
# Arrange
metadata = FakeIngestMetadata.for_state(region="us_nd")
# 5-digit address could be mistaken for a zip code
ingest_person = ingest_info_pb2.StatePerson(current_address="12345 Main 58503")
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build(StatePersonFactory.deserialize)
# Assert
expected_result = entities.StatePerson.new_with_defaults(
current_address="12345 MAIN 58503",
residency_status=ResidencyStatus.PERMANENT,
state_code="US_ND",
)
self.assertEqual(result, expected_result)
def testParseStatePerson_NoiseInPlaceOfResidence_ParsesResidency(self):
# Arrange
metadata = FakeIngestMetadata.for_state(region="us_xx")
ingest_person = ingest_info_pb2.StatePerson(
current_address="transient moves around"
)
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build(StatePersonFactory.deserialize)
# Assert
expected_result = entities.StatePerson.new_with_defaults(
current_address="TRANSIENT MOVES AROUND",
residency_status=ResidencyStatus.TRANSIENT,
state_code="US_XX",
)
self.assertEqual(result, expected_result)
def testParsePerson_NoiseInPlaceOfResidence_ParsesResidencyStatus(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="us_ky_allen", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.Person(
place_of_residence="transient moves around")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
residency_status=ResidencyStatus.TRANSIENT,
region="us_ky_allen",
jurisdiction_id="JURISDICTION_ID",
)
self.assertEqual(result, expected_result)
def testParsePerson_ResidenceAndStatusCombined(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="us_ky_allen", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.Person(
place_of_residence="42164 homeless")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=True,
residency_status=ResidencyStatus.HOMELESS,
region="us_ky_allen",
jurisdiction_id="JURISDICTION_ID",
)
self.assertEqual(result, expected_result)
def testParsePerson_InfersBirthdateFromAge(self, mock_datetime):
# Arrange
mock_datetime.now.return_value = _NOW
metadata = FakeIngestMetadata.for_county(
region="REGION", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.Person(age="27")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
birthdate=datetime(year=_NOW.year - 27, month=1, day=1).date(),
birthdate_inferred_from_age=True,
)
self.assertEqual(result, expected_result)
def testParseBooking(self):
# Arrange
metadata = FakeIngestMetadata.for_county(region="REGION",
ingest_time=_INGEST_TIME)
ingest_booking = ingest_info_pb2.Booking(
booking_id="BOOKING_ID",
admission_date="2/3/1000",
admission_reason="New Commitment",
release_date="1/2/2017",
projected_release_date="5/20/2020",
release_reason="Transfer",
custody_status="Held Elsewhere",
classification="Low",
)
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=date(year=1000, month=2, day=3),
admission_reason=AdmissionReason.NEW_COMMITMENT,
admission_reason_raw_text="NEW COMMITMENT",
admission_date_inferred=False,
release_date=date(year=2017, month=1, day=2),
release_date_inferred=False,
projected_release_date=date(year=2020, month=5, day=20),
release_reason=ReleaseReason.TRANSFER,
release_reason_raw_text="TRANSFER",
custody_status=CustodyStatus.HELD_ELSEWHERE,
custody_status_raw_text="HELD ELSEWHERE",
classification=Classification.LOW,
classification_raw_text="LOW",
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testParsePerson_NotResidentOfState(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="us_ky", jurisdiction_id="JURISDICTION_ID")
# 10011 is in New York
ingest_person = ingest_info_pb2.Person(
place_of_residence="123 Main 10011")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=False,
residency_status=ResidencyStatus.PERMANENT,
region="us_ky",
jurisdiction_id="JURISDICTION_ID",
)
self.assertEqual(result, expected_result)
def testParsePerson_TakesLastZipCodeMatch(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="us_ky_allen", jurisdiction_id="JURISDICTION_ID")
# 5-digit address could be mistaken for a zip code
ingest_person = ingest_info_pb2.Person(
place_of_residence="12345 Main 42164")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=True,
residency_status=ResidencyStatus.PERMANENT,
region="us_ky_allen",
jurisdiction_id="JURISDICTION_ID",
)
self.assertEqual(result, expected_result)
def testConvert_TotalBondWithMultipleBonds_ThrowsException(self):
# Arrange
metadata = FakeIngestMetadata.for_county(
region="REGION",
jurisdiction_id="JURISDICTION_ID",
ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=["BOOKING_ID"])
ingest_info.bookings.add(
booking_id="BOOKING_ID",
total_bond_amount="$100",
charge_ids=["CHARGE_ID", "CHARGE_ID_2"],
)
ingest_info.charges.add(charge_id="CHARGE_ID", bond_id="BOND_ID")
ingest_info.charges.add(charge_id="CHARGE_ID_2", bond_id="BOND_ID_2")
ingest_info.bonds.add(bond_id="BOND_ID")
ingest_info.bonds.add(bond_id="BOND_ID_2")
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)