def testParsesStatePerson(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(
region="us_nd", jurisdiction_id="JURISDICTION_ID")
ingest_person = ingest_info_pb2.StatePerson(
gender="MALE",
full_name="FULL_NAME",
birthdate="12-31-1999",
current_address="NNN\n STREET \t ZIP",
)
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person,
metadata)
result = self.subject.build(StatePersonFactory.deserialize)
# Assert
expected_result = entities.StatePerson.new_with_defaults(
gender=Gender.MALE,
gender_raw_text="MALE",
full_name='{"full_name": "FULL_NAME"}',
birthdate=date(year=1999, month=12, day=31),
birthdate_inferred_from_age=False,
current_address="NNN STREET ZIP",
residency_status=ResidencyStatus.PERMANENT,
state_code="US_ND",
)
self.assertEqual(result, expected_result)
def testParsesPerson(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(
region="REGION", jurisdiction_id="JURISDICTION_ID"
)
ingest_person = ingest_info_pb2.Person(
full_name="FULL_NAME",
birthdate="12-31-1999",
gender="MALE",
race="WHITE",
ethnicity="HISPANIC",
place_of_residence="NNN\n STREET \t ZIP",
)
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
full_name='{{"full_name": "{}"}}'.format("FULL_NAME"),
birthdate=date(year=1999, month=12, day=31),
birthdate_inferred_from_age=False,
gender=Gender.MALE,
gender_raw_text="MALE",
race=Race.WHITE,
race_raw_text="WHITE",
ethnicity=Ethnicity.HISPANIC,
ethnicity_raw_text="HISPANIC",
residency_status=ResidencyStatus.PERMANENT,
region="REGION",
jurisdiction_id="JURISDICTION_ID",
)
self.assertEqual(result, expected_result)
def testParsesStatePerson(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(
region='us_nd', jurisdiction_id='JURISDICTION_ID')
ingest_person = ingest_info_pb2.StatePerson(
gender='MALE',
full_name='FULL_NAME',
birthdate='12-31-1999',
current_address='NNN\n STREET \t ZIP')
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person,
metadata)
result = self.subject.build()
# Assert
expected_result = entities.StatePerson.new_with_defaults(
gender=Gender.MALE,
gender_raw_text='MALE',
full_name='{"full_name": "FULL_NAME"}',
birthdate=date(year=1999, month=12, day=31),
birthdate_inferred_from_age=False,
current_address='NNN STREET ZIP',
residency_status=ResidencyStatus.PERMANENT,
)
self.assertEqual(result, expected_result)
def testConvert_PersonInferredBooking(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add()
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(
bookings=[
Booking.new_with_defaults(
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
admission_date=_INGEST_TIME.date(),
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
)
]
)
]
self.assertEqual(result, expected_result)
def testParseBooking_MapAcrossFields(self):
# Arrange
overrides_builder = EnumOverrides.Builder()
overrides_builder.add("WORK RELEASE", Classification.WORK_RELEASE,
AdmissionReason)
overrides_builder.add("transfer", AdmissionReason.TRANSFER,
CustodyStatus)
metadata = IngestMetadata.new_with_defaults(
ingest_time=_INGEST_TIME, enum_overrides=overrides_builder.build())
ingest_booking = ingest_info_pb2.Booking(
admission_reason="work release", custody_status="transfer")
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_reason=AdmissionReason.TRANSFER,
admission_reason_raw_text="WORK RELEASE",
admission_date_inferred=True,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
custody_status_raw_text="TRANSFER",
classification=Classification.WORK_RELEASE,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testParseSentence_completionDateWithoutStatus_marksAsCompleted(self):
# Arrange
ingest_sentence = ingest_info_pb2.Sentence(
sentence_id="SENTENCE_ID",
completion_date="2018-1-1",
)
# Act
sentence_builder = entities.Sentence.builder()
sentence.copy_fields_to_builder(
sentence_builder,
ingest_sentence,
IngestMetadata.new_with_defaults(
ingest_time=datetime.datetime(year=2018, month=6, day=8)
),
)
result = sentence_builder.build()
# Assert
expected_result = entities.Sentence.new_with_defaults(
external_id="SENTENCE_ID",
completion_date=datetime.date(year=2018, month=1, day=1),
status=SentenceStatus.COMPLETED,
)
self.assertEqual(result, expected_result)
def testParsePerson_RaceIsEthnicity(self):
class ScraperWithDefaultOverrides(BaseScraper):
"""Class created so BaseScraper's enum_overrides can be used."""
def __init__(self): # pylint: disable=super-init-not-called
self.region = Mock()
def populate_data(self, _, __, ___):
pass
# Arrange
metadata = IngestMetadata.new_with_defaults(
enum_overrides=ScraperWithDefaultOverrides().get_enum_overrides()
)
ingest_person = ingest_info_pb2.Person(race="HISPANIC")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
ethnicity=Ethnicity.HISPANIC,
race_raw_text="HISPANIC",
)
self.assertEqual(result, expected_result)
def testConvert_TotalBondNoCharge_CreatesChargeWithTotalBondAmount(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=['BOOKING_ID'])
ingest_info.bookings.add(booking_id='BOOKING_ID',
total_bond_amount='$100')
# Act
result = self._convert_and_throw_on_errors(ingest_info, metadata)
# Assert
expected_result = [
Person.new_with_defaults(bookings=[
Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
first_seen_time=_INGEST_TIME,
last_seen_time=_INGEST_TIME,
external_id='BOOKING_ID',
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
charges=[
Charge.new_with_defaults(
status=ChargeStatus.PRESENT_WITHOUT_INFO,
bond=Bond.new_with_defaults(
status=BondStatus.PRESENT_WITHOUT_INFO,
bond_type=BondType.CASH,
amount_dollars=100))
])
])
]
self.assertEqual(result, expected_result)
def testParseBooking_releaseDateInFuture_setAsProjected(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_booking = ingest_info_pb2.Booking(
booking_id="BOOKING_ID",
admission_date="2/3/1000",
release_date="1/2/2020",
)
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=date(year=1000, month=2, day=3),
admission_date_inferred=False,
release_date=None,
release_date_inferred=None,
projected_release_date=date(year=2020, month=1, day=2),
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testParseBooking_releaseDateWithoutStatus_marksAsReleased(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_booking = ingest_info_pb2.Booking(
booking_id="BOOKING_ID",
release_date="2/1/2019",
)
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
external_id="BOOKING_ID",
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
release_date=date(year=2019, month=2, day=1),
release_date_inferred=False,
custody_status=CustodyStatus.RELEASED,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
)
self.assertEqual(result, expected_result)
def testCompletionDateInFuture(self):
# Arrange
ingest_sentence = ingest_info_pb2.Sentence(
sentence_id="SENTENCE_ID",
completion_date="2020-1-1",
)
# Act
sentence_builder = entities.Sentence.builder()
sentence.copy_fields_to_builder(
sentence_builder,
ingest_sentence,
IngestMetadata.new_with_defaults(
ingest_time=datetime.datetime(year=2018, month=6, day=8)
),
)
result = sentence_builder.build()
# Assert
expected_result = entities.Sentence.new_with_defaults(
external_id="SENTENCE_ID",
completion_date=None,
projected_completion_date=datetime.date(year=2020, month=1, day=1),
status=SentenceStatus.PRESENT_WITHOUT_INFO,
)
self.assertEqual(result, expected_result)
def testParseSentence(self):
# Arrange
ingest_sentence = ingest_info_pb2.Sentence(
sentence_id="SENTENCE_ID",
min_length="1",
date_imposed="2000-1-1",
post_release_supervision_length="",
)
# Act
sentence_builder = entities.Sentence.builder()
sentence.copy_fields_to_builder(
sentence_builder,
ingest_sentence,
IngestMetadata.new_with_defaults(
ingest_time=datetime.datetime(year=2018, month=6, day=8)
),
)
result = sentence_builder.build()
# Assert
expected_result = entities.Sentence.new_with_defaults(
external_id="SENTENCE_ID",
min_length_days=1,
post_release_supervision_length_days=0,
date_imposed=datetime.date(year=2000, month=1, day=1),
status=SentenceStatus.PRESENT_WITHOUT_INFO,
)
self.assertEqual(result, expected_result)
def testParsePerson_WithSurnameAndFullname_ThrowsException(self):
# Arrange
metadata = IngestMetadata.new_with_defaults()
ingest_person = ingest_info_pb2.Person(full_name="LAST,FIRST", surname="LAST")
# Arrange + Act
with self.assertRaises(ValueError):
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
def test_write_preexisting_person(self):
# Arrange
most_recent_scrape_time = (SCRAPER_START_DATETIME + timedelta(days=1))
metadata = IngestMetadata.new_with_defaults(
region=REGION_1,
jurisdiction_id=JURISDICTION_ID,
ingest_time=most_recent_scrape_time)
schema_booking = schema.Booking(
booking_id=BOOKING_ID,
external_id=EXTERNAL_BOOKING_ID,
admission_date_inferred=True,
custody_status=CustodyStatus.IN_CUSTODY.value,
last_seen_time=SCRAPER_START_DATETIME,
first_seen_time=SCRAPER_START_DATETIME)
schema_person = schema.Person(
person_id=PERSON_ID,
jurisdiction_id=JURISDICTION_ID,
external_id=EXTERNAL_PERSON_ID,
region=REGION_1,
bookings=[schema_booking])
session = SessionFactory.for_schema_base(JailsBase)
session.add(schema_person)
session.commit()
ingest_info = IngestInfo()
ingest_info.people.add(full_name=FULL_NAME_1,
person_id=EXTERNAL_PERSON_ID,
booking_ids=[EXTERNAL_BOOKING_ID])
ingest_info.bookings.add(
booking_id=EXTERNAL_BOOKING_ID,
custody_status='IN CUSTODY',
)
# Act
persistence.write(ingest_info, metadata)
# Assert
expected_booking = county_entities.Booking.new_with_defaults(
booking_id=BOOKING_ID,
external_id=EXTERNAL_BOOKING_ID,
admission_date_inferred=True,
custody_status=CustodyStatus.IN_CUSTODY,
custody_status_raw_text=BOOKING_CUSTODY_STATUS.upper(),
last_seen_time=most_recent_scrape_time,
first_seen_time=SCRAPER_START_DATETIME)
expected_person = county_entities.Person.new_with_defaults(
person_id=PERSON_ID,
external_id=EXTERNAL_PERSON_ID,
region=REGION_1,
jurisdiction_id=JURISDICTION_ID,
bookings=[expected_booking])
self.assertEqual([expected_person],
county_dao.read_people(
SessionFactory.for_schema_base(JailsBase)))
def testConvert_CannotConvertField_RaisesValueError(self):
# Arrange
metadata = IngestMetadata.new_with_defaults()
ingest_info = IngestInfo()
ingest_info.people.add(birthdate='NOT_A_DATE')
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)
def testConvert_CannotConvertField_RaisesValueError(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(
system_level=SystemLevel.STATE)
ingest_info = IngestInfo()
ingest_info.state_people.add(birthdate="NOT_A_DATE")
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)
def testParseBooking_facilityId_doesNotExist(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME, )
ingest_booking = ingest_info_pb2.Booking(
facility_id="1234567890123456" # invalid FID
)
with self.assertRaises(ValueError):
booking.copy_fields_to_builder(self.subject, ingest_booking,
metadata)
def testConvert_MultipleOpenBookings_RaisesValueError(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=["BOOKING_ID1", "BOOKING_ID2"])
ingest_info.bookings.add(booking_id="BOOKING_ID1", admission_date="3/14/2020")
ingest_info.bookings.add(booking_id="BOOKING_ID2", admission_date="3/16/2020")
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)
def testParsePerson_NoNames_FullNameIsNone(self):
# Arrange
metadata = IngestMetadata.new_with_defaults()
ingest_person = ingest_info_pb2.Person(person_id="1234")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(external_id="1234")
self.assertEqual(result, expected_result)
def testParsePerson_NoiseInPlaceOfResidence_ParsesResidencyStatus(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region='us_ky_allen')
ingest_person = ingest_info_pb2.Person(
place_of_residence='transient moves around')
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
residency_status=ResidencyStatus.TRANSIENT, region='us_ky_allen')
self.assertEqual(result, expected_result)
def testParsePerson_InfersBirthdateFromAge(self, mock_datetime):
# Arrange
mock_datetime.now.return_value = _NOW
metadata = IngestMetadata.new_with_defaults()
ingest_person = ingest_info_pb2.Person(age='27')
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
birthdate=datetime(year=_NOW.year - 27, month=1, day=1).date(),
birthdate_inferred_from_age=True)
self.assertEqual(result, expected_result)
def testParsePerson_ResidenceAndStatusCombined(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region="us_ky_allen")
ingest_person = ingest_info_pb2.Person(place_of_residence="42164 homeless")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=True,
residency_status=ResidencyStatus.HOMELESS,
region="us_ky_allen",
)
self.assertEqual(result, expected_result)
def test_write_noPeople(self):
# Arrange
most_recent_scrape_time = (SCRAPER_START_DATETIME + timedelta(days=1))
metadata = IngestMetadata.new_with_defaults(
region=REGION_1,
jurisdiction_id=JURISDICTION_ID,
ingest_time=most_recent_scrape_time)
ingest_info = IngestInfo()
# Act
persistence.write(ingest_info, metadata)
# Assert
people = county_dao.read_people(
SessionFactory.for_schema_base(JailsBase))
self.assertFalse(people)
def testConvert_TotalBondWithMultipleBonds_ThrowsException(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME)
ingest_info = IngestInfo()
ingest_info.people.add(booking_ids=['BOOKING_ID'])
ingest_info.bookings.add(booking_id='BOOKING_ID',
total_bond_amount='$100',
charge_ids=['CHARGE_ID', 'CHARGE_ID_2'])
ingest_info.charges.add(charge_id='CHARGE_ID', bond_id='BOND_ID')
ingest_info.charges.add(charge_id='CHARGE_ID_2', bond_id='BOND_ID_2')
ingest_info.bonds.add(bond_id='BOND_ID')
ingest_info.bonds.add(bond_id='BOND_ID_2')
# Act + Assert
with self.assertRaises(ValueError):
self._convert_and_throw_on_errors(ingest_info, metadata)
def testParseBooking_SetsDefaults(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(ingest_time=_INGEST_TIME, )
ingest_booking = ingest_info_pb2.Booking()
# Act
booking.copy_fields_to_builder(self.subject, ingest_booking, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Booking.new_with_defaults(
admission_date=_INGEST_TIME.date(),
admission_date_inferred=True,
last_seen_time=_INGEST_TIME,
first_seen_time=_INGEST_TIME,
custody_status=CustodyStatus.PRESENT_WITHOUT_INFO)
self.assertEqual(result, expected_result)
def testParseStatePerson_TakesLastZipCodeMatch(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region='us_nd')
# 5-digit address could be mistaken for a zip code
ingest_person = ingest_info_pb2.StatePerson(
current_address='12345 Main 58503')
# Act
state_person.copy_fields_to_builder(self.subject, ingest_person,
metadata)
result = self.subject.build()
# Assert
expected_result = entities.StatePerson.new_with_defaults(
current_address='12345 MAIN 58503',
residency_status=ResidencyStatus.PERMANENT)
self.assertEqual(result, expected_result)
def testParsePerson_NotResidentOfCounty(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region='us_ky_allen')
# 40601 is in Frankfort
ingest_person = ingest_info_pb2.Person(
place_of_residence='123 Main 40601')
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=False,
residency_status=ResidencyStatus.PERMANENT,
region='us_ky_allen')
self.assertEqual(result, expected_result)
def testParsePerson_ResidentOfCounty(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region="us_ky_allen")
# 42164 is in Allen
ingest_person = ingest_info_pb2.Person(place_of_residence="123 Main 42164")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=True,
residency_status=ResidencyStatus.PERMANENT,
region="us_ky_allen",
)
self.assertEqual(result, expected_result)
def testParsePerson_NotResidentOfState(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region="us_ky")
# 10011 is in New York
ingest_person = ingest_info_pb2.Person(place_of_residence="123 Main 10011")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=False,
residency_status=ResidencyStatus.PERMANENT,
region="us_ky",
)
self.assertEqual(result, expected_result)
def testParsePerson_TakesLastZipCodeMatch(self):
# Arrange
metadata = IngestMetadata.new_with_defaults(region="us_ky_allen")
# 5-digit address could be mistaken for a zip code
ingest_person = ingest_info_pb2.Person(place_of_residence="12345 Main 42164")
# Act
person.copy_fields_to_builder(self.subject, ingest_person, metadata)
result = self.subject.build()
# Assert
expected_result = entities.Person.new_with_defaults(
resident_of_region=True,
residency_status=ResidencyStatus.PERMANENT,
region="us_ky_allen",
)
self.assertEqual(result, expected_result)