def test_mergeFlatFields_incompleteIncarcerationPeriods(self):
ingested_entity = schema.StateIncarcerationPeriod(
incarceration_period_id=_ID,
external_id=_EXTERNAL_ID,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY, admission_date=_DATE_1,
admission_reason=
StateIncarcerationPeriodAdmissionReason.NEW_ADMISSION)
db_entity = schema.StateIncarcerationPeriod(
external_id=_EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY, release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER)
expected_incarceration_period = \
schema.StateIncarcerationPeriod(
incarceration_period_id=_ID,
external_id=_EXTERNAL_ID + '|' + _EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY, admission_date=_DATE_1,
admission_reason=
StateIncarcerationPeriodAdmissionReason.NEW_ADMISSION,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER)
self.assert_schema_objects_equal(
expected_incarceration_period,
merge_incomplete_periods(new_entity=ingested_entity,
old_entity=db_entity))
def test_mergeIncarcerationPeriods_doNotMergeWithPlaceholder(self) -> None:
incarceration_period = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
)
placeholder_incarceration_period = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ingested_incarceration_periods = [
incarceration_period,
placeholder_incarceration_period,
]
expected_periods = [
incarceration_period, placeholder_incarceration_period
]
merged_periods = _merge_incarceration_periods_helper(
ingested_incarceration_periods)
self.assert_schema_object_lists_equal(expected_periods, merged_periods)
def test_mergeIncarcerationPeriods_doNotMergeNonConsecutiveSequences(
self) -> None:
incarceration_period_1 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
)
incarceration_period_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_3,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
ingested_incarceration_periods = [
incarceration_period_1,
incarceration_period_2,
]
expected_incarceration_periods = [
incarceration_period_1,
incarceration_period_2,
]
merged_periods = _merge_incarceration_periods_helper(
ingested_incarceration_periods)
self.assert_schema_object_lists_equal(expected_incarceration_periods,
merged_periods)
incarceration_period_2.external_id = _EXTERNAL_ID_2
expected_merged_incarceration_period_1 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID + "|" + _EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
expected_incarceration_periods = [
expected_merged_incarceration_period_1
]
merged_periods = _merge_incarceration_periods_helper(
ingested_incarceration_periods)
self.assert_schema_object_lists_equal(expected_incarceration_periods,
merged_periods)
def test_transformToHoldsOpenPeriod(self) -> None:
# Arrange
ip = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id="123-1",
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
TEMPORARY_CUSTODY.value,
admission_reason_raw_text="PV",
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
expected_ip = attr.evolve(self.to_entity(ip))
ips = [ip]
expected_ips = [expected_ip]
# Act
_update_temporary_holds_helper(ips, self.overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def test_transformToHolds_nonTransferReason(self) -> None:
# Arrange
ip = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION.value,
admission_reason_raw_text="PV",
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ip_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION.value,
release_date=_DATE_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.STATE_PRISON.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
expected_ip = attr.evolve(
self.to_entity(ip),
admission_reason=StateIncarcerationPeriodAdmissionReason.
TEMPORARY_CUSTODY,
release_reason=StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY,
)
expected_ip_2 = attr.evolve(self.to_entity(ip_2))
ips = [ip, ip_2]
expected_ips = [expected_ip, expected_ip_2]
overrides = self.create_fake_nd_region().get_enum_overrides()
# Act
_update_temporary_holds_helper(ips, overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def generate_incarceration_period(person,
**kwargs) -> schema.StateIncarcerationPeriod:
args = {
"status": StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO.value,
"state_code": _STATE_CODE,
"incarceration_type": StateIncarcerationType.STATE_PRISON.value,
}
args.update(kwargs)
return schema.StateIncarcerationPeriod(person=person, **args)
def test_getRootEntity_allPlaceholders_raises(self) -> None:
placeholder_incarceration_period = schema.StateIncarcerationPeriod()
placeholder_incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_periods=[placeholder_incarceration_period])
placeholder_sentence_group = schema.StateSentenceGroup(
incarceration_sentences=[placeholder_incarceration_sentence])
placeholder_person = schema.StatePerson(
sentence_groups=[placeholder_sentence_group])
with pytest.raises(EntityMatchingError):
get_root_entity_cls([placeholder_person])
def test_completeEnumSet_admittedForRevocation(self):
period = schema.StateIncarcerationPeriod()
for admission_reason in StateIncarcerationPeriodAdmissionReason:
period.admission_reason = admission_reason.value
is_revocation_admission(
StateIncarcerationPeriodAdmissionReason.
parse_from_canonical_string(period.admission_reason))
is_revocation_admission(
StateIncarcerationPeriodAdmissionReason.
parse_from_canonical_string(None))
def generate_test_incarceration_period(person_id, incarceration_incidents,
parole_decisions) -> \
state_schema.StateIncarcerationPeriod:
instance = state_schema.StateIncarcerationPeriod(
incarceration_period_id=5555,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY.value,
state_code='us_ca',
person_id=person_id,
incarceration_incidents=incarceration_incidents,
parole_decisions=parole_decisions,
)
return instance
def test_completeEnumSet_admittedForCommitmentFromSupervision(
self) -> None:
period = schema.StateIncarcerationPeriod()
for admission_reason in StateIncarcerationPeriodAdmissionReason:
period.admission_reason = admission_reason.value
is_commitment_from_supervision(
StateIncarcerationPeriodAdmissionReason.
parse_from_canonical_string(period.admission_reason, ),
allow_ingest_only_enum_values=True,
)
is_commitment_from_supervision(
StateIncarcerationPeriodAdmissionReason.
parse_from_canonical_string(None))
def test_getRootEntity(self) -> None:
# Arrange
incarceration_incident = schema.StateIncarcerationIncident(
external_id=_EXTERNAL_ID)
placeholder_incarceration_period = schema.StateIncarcerationPeriod(
incarceration_incidents=[incarceration_incident])
placeholder_incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_periods=[placeholder_incarceration_period])
placeholder_sentence_group = schema.StateSentenceGroup(
sentence_group_id=None,
incarceration_sentences=[placeholder_incarceration_sentence],
)
person = schema.StatePerson(
sentence_groups=[placeholder_sentence_group])
# Act
root_entity_cls = get_root_entity_cls([person])
# Assert
self.assertEqual(schema.StateIncarcerationIncident, root_entity_cls)
def test_transformToHoldsOpenPeriod(self):
# Arrange
ip = schema.StateIncarcerationPeriod(
external_id='123-1',
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.TEMPORARY_CUSTODY.value,
admission_reason_raw_text='PV',
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value
)
expected_ip = attr.evolve(self.to_entity(ip))
ips = [ip]
expected_ips = [expected_ip]
overrides = self.create_fake_nd_region().get_enum_overrides()
# Act
_update_temporary_holds_helper(ips, overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def test_transformToHolds_takeAdmissionReasonFromConsecutive(self):
# Arrange
# Too long of a time gap between date_1 and date_2 to be
# considered consecutive
date_1 = _DATE_1
date_2 = date_1 + datetime.timedelta(days=3)
date_3 = date_2 + datetime.timedelta(days=2)
ip = schema.StateIncarcerationPeriod(
admission_date=date_1,
admission_reason=
StateIncarcerationPeriodAdmissionReason.PAROLE_REVOCATION.value,
admission_reason_raw_text='PV',
release_date=date_1,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.EXTERNAL_UNKNOWN.value
)
ip_2 = schema.StateIncarcerationPeriod(
admission_date=date_2,
admission_reason=
StateIncarcerationPeriodAdmissionReason.NEW_ADMISSION.value,
release_date=date_2,
admission_reason_raw_text='ADM',
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
)
ip_3 = schema.StateIncarcerationPeriod(
admission_date=date_3,
admission_reason=
StateIncarcerationPeriodAdmissionReason.TRANSFER.value,
admission_reason_raw_text='INT',
release_date=date_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.STATE_PRISON.value,
)
expected_ip = attr.evolve(
self.to_entity(ip),
admission_reason=
StateIncarcerationPeriodAdmissionReason.TEMPORARY_CUSTODY,
release_reason=
StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY)
expected_ip_2 = attr.evolve(
self.to_entity(ip_2),
admission_reason=
StateIncarcerationPeriodAdmissionReason.TEMPORARY_CUSTODY,
release_reason=
StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY)
expected_ip_3 = attr.evolve(
self.to_entity(ip_3),
admission_reason=
StateIncarcerationPeriodAdmissionReason.NEW_ADMISSION)
ips = [ip_2, ip, ip_3]
expected_ips = [expected_ip, expected_ip_2, expected_ip_3]
overrides = self.create_fake_nd_region().get_enum_overrides()
# Act
_update_temporary_holds_helper(ips, overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def test_transformToHolds(self):
# Arrange
ip = schema.StateIncarcerationPeriod(
admission_date=_DATE_1,
admission_reason=
StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION.value,
admission_reason_raw_text='PV',
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.EXTERNAL_UNKNOWN.value
)
ip_2 = schema.StateIncarcerationPeriod(
admission_date=_DATE_2,
admission_reason=
StateIncarcerationPeriodAdmissionReason.TRANSFER.value,
admission_reason_raw_text='INT',
release_date=_DATE_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
)
ip_3 = schema.StateIncarcerationPeriod(
admission_date=_DATE_3,
admission_reason=
StateIncarcerationPeriodAdmissionReason.TRANSFER.value,
admission_reason_raw_text='INT',
release_date=_DATE_4,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.STATE_PRISON.value,
)
ip_4 = schema.StateIncarcerationPeriod(
admission_date=_DATE_4,
admission_reason=
StateIncarcerationPeriodAdmissionReason.TRANSFER.value,
admission_reason_raw_text='INT',
release_date=_DATE_5,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
)
expected_ip = attr.evolve(
self.to_entity(ip),
admission_reason=
StateIncarcerationPeriodAdmissionReason.TEMPORARY_CUSTODY,
release_reason=
StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY)
expected_ip_2 = attr.evolve(
self.to_entity(ip_2),
admission_reason=
StateIncarcerationPeriodAdmissionReason.TEMPORARY_CUSTODY,
release_reason=
StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY)
expected_ip_3 = attr.evolve(
self.to_entity(ip_3),
admission_reason=
StateIncarcerationPeriodAdmissionReason.PAROLE_REVOCATION)
expected_ip_4 = attr.evolve(self.to_entity(ip_4))
ips = [ip_2, ip_4, ip, ip_3]
expected_ips = [
expected_ip, expected_ip_2, expected_ip_3, expected_ip_4]
overrides = self.create_fake_nd_region().get_enum_overrides()
# Act
_update_temporary_holds_helper(ips, overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def build_incarceration_pipeline_data_dict(self,
fake_person_id: int,
state_code: str = "US_XX"):
"""Builds a data_dict for a basic run of the pipeline."""
fake_person = schema.StatePerson(
state_code=state_code,
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT,
)
persons_data = [normalized_database_base_dict(fake_person)]
race_1 = schema.StatePersonRace(
person_race_id=111,
state_code=state_code,
race=Race.BLACK,
person_id=fake_person_id,
)
race_2 = schema.StatePersonRace(
person_race_id=111,
state_code=state_code,
race=Race.WHITE,
person_id=fake_person_id,
)
races_data = normalized_database_base_dict_list([race_1, race_2])
ethnicity = schema.StatePersonEthnicity(
person_ethnicity_id=111,
state_code=state_code,
ethnicity=Ethnicity.HISPANIC,
person_id=fake_person_id,
)
ethnicity_data = normalized_database_base_dict_list([ethnicity])
sentence_group = schema.StateSentenceGroup(
sentence_group_id=98765,
state_code=state_code,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
person_id=fake_person_id,
)
initial_incarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=1111,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code=state_code,
county_code="124",
facility="San Quentin",
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2008, 11, 20),
release_date=date(2010, 12, 4),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id,
)
first_reincarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=2222,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code=state_code,
county_code="124",
facility="San Quentin",
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2011, 4, 5),
release_date=date(2014, 4, 14),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id,
)
subsequent_reincarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=3333,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
state_code=state_code,
county_code="124",
facility="San Quentin",
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2017, 1, 4),
person_id=fake_person_id,
)
incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_sentence_id=1111,
state_code=state_code,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
sentence_group_id=sentence_group.sentence_group_id,
incarceration_periods=[
initial_incarceration,
first_reincarceration,
subsequent_reincarceration,
],
person_id=fake_person_id,
)
supervision_sentence = schema.StateSupervisionSentence(
supervision_sentence_id=123,
state_code=state_code,
sentence_group_id=sentence_group.sentence_group_id,
person_id=fake_person_id,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
)
sentence_group.incarceration_sentences = [incarceration_sentence]
sentence_group.supervision_sentences = [supervision_sentence]
sentence_group_data = [normalized_database_base_dict(sentence_group)]
incarceration_sentence_data = [
normalized_database_base_dict(incarceration_sentence)
]
supervision_sentence_data = [
normalized_database_base_dict(supervision_sentence)
]
incarceration_periods_data = [
normalized_database_base_dict(initial_incarceration),
normalized_database_base_dict(first_reincarceration),
normalized_database_base_dict(subsequent_reincarceration),
]
state_incarceration_sentence_incarceration_period_association = [
{
"incarceration_period_id":
initial_incarceration.incarceration_period_id,
"incarceration_sentence_id":
incarceration_sentence.incarceration_sentence_id,
},
{
"incarceration_period_id":
first_reincarceration.incarceration_period_id,
"incarceration_sentence_id":
incarceration_sentence.incarceration_sentence_id,
},
{
"incarceration_period_id":
subsequent_reincarceration.incarceration_period_id,
"incarceration_sentence_id":
incarceration_sentence.incarceration_sentence_id,
},
]
fake_person_id_to_county_query_result = [{
"state_code":
state_code,
"person_id":
fake_person_id,
"county_of_residence":
_COUNTY_OF_RESIDENCE,
}]
us_mo_sentence_status_data: List[Dict[str, Any]] = [{
"state_code":
"US_MO",
"person_id":
fake_person_id,
"sentence_external_id":
"XXX",
"sentence_status_external_id":
"YYY",
"status_code":
"ZZZ",
"status_date":
"not_a_date",
"status_description":
"XYZ",
}]
incarceration_period_judicial_district_association_data = [{
"state_code":
state_code,
"person_id":
fake_person_id,
"incarceration_period_id":
123,
"judicial_district_code":
"NW",
}]
state_race_ethnicity_population_count_data = [{
"state_code":
state_code,
"race_or_ethnicity":
"BLACK",
"population_count":
1,
"representation_priority":
1,
}]
data_dict = self._default_data_dict()
data_dict_overrides = {
schema.StatePerson.__tablename__:
persons_data,
schema.StatePersonRace.__tablename__:
races_data,
schema.StatePersonEthnicity.__tablename__:
ethnicity_data,
schema.StateSentenceGroup.__tablename__:
sentence_group_data,
schema.StateIncarcerationSentence.__tablename__:
incarceration_sentence_data,
schema.StateSupervisionSentence.__tablename__:
supervision_sentence_data,
schema.StateIncarcerationPeriod.__tablename__:
incarceration_periods_data,
schema.state_incarceration_sentence_incarceration_period_association_table.name:
state_incarceration_sentence_incarceration_period_association,
"persons_to_recent_county_of_residence":
fake_person_id_to_county_query_result,
"incarceration_period_judicial_district_association":
incarceration_period_judicial_district_association_data,
"state_race_ethnicity_population_counts":
state_race_ethnicity_population_count_data,
"us_mo_sentence_statuses":
us_mo_sentence_status_data,
}
data_dict.update(data_dict_overrides)
return data_dict
def build_incarceration_pipeline_data_dict_no_incarceration(
self, fake_person_id: int):
"""Builds a data_dict for a run of the pipeline where the person has no incarceration."""
fake_person_1 = schema.StatePerson(
state_code="US_XX",
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT,
)
fake_person_id_2 = 6789
fake_person_2 = schema.StatePerson(
state_code="US_XX",
person_id=fake_person_id_2,
gender=Gender.FEMALE,
birthdate=date(1990, 1, 1),
residency_status=ResidencyStatus.PERMANENT,
)
persons_data = [
normalized_database_base_dict(fake_person_1),
normalized_database_base_dict(fake_person_2),
]
sentence_group = schema.StateSentenceGroup(
sentence_group_id=111,
state_code="US_XX",
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
person_id=fake_person_id,
)
incarceration_period = schema.StateIncarcerationPeriod(
incarceration_period_id=1111,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code="US_XX",
county_code="124",
facility="San Quentin",
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2008, 11, 20),
release_date=date(2010, 12, 4),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id,
)
incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_sentence_id=1111,
state_code="US_XX",
sentence_group_id=sentence_group.sentence_group_id,
incarceration_periods=[incarceration_period],
person_id=fake_person_id,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
)
supervision_sentence = schema.StateSupervisionSentence(
supervision_sentence_id=123,
state_code="US_XX",
person_id=fake_person_id,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
)
sentence_group.incarceration_sentences = [incarceration_sentence]
sentence_group_data = [normalized_database_base_dict(sentence_group)]
incarceration_sentence_data = [
normalized_database_base_dict(incarceration_sentence)
]
supervision_sentence_data = [
normalized_database_base_dict(supervision_sentence)
]
incarceration_periods_data = [
normalized_database_base_dict(incarceration_period)
]
state_incarceration_sentence_incarceration_period_association = [
{
"incarceration_period_id":
incarceration_period.incarceration_period_id,
"incarceration_sentence_id":
incarceration_sentence.incarceration_sentence_id,
},
]
fake_person_id_to_county_query_result = [{
"state_code":
"US_XX",
"person_id":
fake_person_id,
"county_of_residence":
_COUNTY_OF_RESIDENCE,
}]
us_mo_sentence_status_data: List[Dict[str, Any]] = [{
"state_code":
"US_MO",
"person_id":
fake_person_id,
"sentence_external_id":
"XXX",
"sentence_status_external_id":
"YYY",
"status_code":
"ZZZ",
"status_date":
"not_a_date",
"status_description":
"XYZ",
}]
incarceration_period_judicial_district_association_data = [{
"state_code":
"US_XX",
"person_id":
fake_person_id,
"incarceration_period_id":
123,
"judicial_district_code":
"NW",
}]
state_race_ethnicity_population_count_data = [{
"state_code":
"US_XX",
"race_or_ethnicity":
"BLACK",
"population_count":
1,
"representation_priority":
1,
}]
data_dict = self._default_data_dict()
data_dict_overrides = {
schema.StatePerson.__tablename__:
persons_data,
schema.StateSentenceGroup.__tablename__:
sentence_group_data,
schema.StateIncarcerationSentence.__tablename__:
incarceration_sentence_data,
schema.StateSupervisionSentence.__tablename__:
supervision_sentence_data,
schema.StateIncarcerationPeriod.__tablename__:
incarceration_periods_data,
schema.state_incarceration_sentence_incarceration_period_association_table.name:
state_incarceration_sentence_incarceration_period_association,
"persons_to_recent_county_of_residence":
fake_person_id_to_county_query_result,
"incarceration_period_judicial_district_association":
incarceration_period_judicial_district_association_data,
"state_race_ethnicity_population_counts":
state_race_ethnicity_population_count_data,
"us_mo_sentence_statuses":
us_mo_sentence_status_data,
}
data_dict.update(data_dict_overrides)
return data_dict
def testIncarcerationPipeline(self):
fake_person_id = 12345
fake_person = schema.StatePerson(
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
persons_data = [normalized_database_base_dict(fake_person)]
race_1 = schema.StatePersonRace(person_race_id=111,
state_code='CA',
race=Race.BLACK,
person_id=fake_person_id)
race_2 = schema.StatePersonRace(person_race_id=111,
state_code='ND',
race=Race.WHITE,
person_id=fake_person_id)
races_data = normalized_database_base_dict_list([race_1, race_2])
ethnicity = schema.StatePersonEthnicity(person_ethnicity_id=111,
state_code='CA',
ethnicity=Ethnicity.HISPANIC,
person_id=fake_person_id)
ethnicity_data = normalized_database_base_dict_list([ethnicity])
sentence_group = schema.StateSentenceGroup(sentence_group_id=111,
person_id=fake_person_id)
initial_incarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=1111,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code='CA',
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2008, 11, 20),
release_date=date(2010, 12, 4),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id,
)
first_reincarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=2222,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code='CA',
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2011, 4, 5),
release_date=date(2014, 4, 14),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id)
subsequent_reincarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=3333,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
state_code='CA',
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2017, 1, 4),
person_id=fake_person_id)
incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_sentence_id=1111,
sentence_group_id=sentence_group.sentence_group_id,
incarceration_periods=[
initial_incarceration, first_reincarceration,
subsequent_reincarceration
],
person_id=fake_person_id)
supervision_sentence = schema.StateSupervisionSentence(
supervision_sentence_id=123, person_id=fake_person_id)
sentence_group.incarceration_sentences = [incarceration_sentence]
sentence_group_data = [normalized_database_base_dict(sentence_group)]
incarceration_sentence_data = [
normalized_database_base_dict(incarceration_sentence)
]
supervision_sentence_data = [
normalized_database_base_dict(supervision_sentence)
]
incarceration_periods_data = [
normalized_database_base_dict(initial_incarceration),
normalized_database_base_dict(first_reincarceration),
normalized_database_base_dict(subsequent_reincarceration)
]
state_incarceration_sentence_incarceration_period_association = [
{
'incarceration_period_id':
initial_incarceration.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
{
'incarceration_period_id':
first_reincarceration.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
{
'incarceration_period_id':
subsequent_reincarceration.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
]
data_dict = {
schema.StatePerson.__tablename__:
persons_data,
schema.StatePersonRace.__tablename__:
races_data,
schema.StatePersonEthnicity.__tablename__:
ethnicity_data,
schema.StateSentenceGroup.__tablename__:
sentence_group_data,
schema.StateIncarcerationSentence.__tablename__:
incarceration_sentence_data,
schema.StateSupervisionSentence.__tablename__:
supervision_sentence_data,
schema.StateIncarcerationPeriod.__tablename__:
incarceration_periods_data,
schema.state_incarceration_sentence_incarceration_period_association_table.name:
state_incarceration_sentence_incarceration_period_association,
schema.state_supervision_sentence_incarceration_period_association_table.name:
[{}]
}
test_pipeline = TestPipeline()
# Get StatePersons
persons = (test_pipeline
| 'Load Persons' >> extractor_utils.BuildRootEntity(
dataset=None,
data_dict=data_dict,
root_schema_class=schema.StatePerson,
root_entity_class=entities.StatePerson,
unifying_id_field='person_id',
build_related_entities=True))
# Get StateSentenceGroups
sentence_groups = (
test_pipeline
| 'Load StateSentencegroups' >> extractor_utils.BuildRootEntity(
dataset=None,
data_dict=data_dict,
root_schema_class=schema.StateSentenceGroup,
root_entity_class=entities.StateSentenceGroup,
unifying_id_field='person_id',
build_related_entities=True))
# Get StateIncarcerationSentences
incarceration_sentences = (
test_pipeline | 'Load StateIncarcerationSentences' >>
extractor_utils.BuildRootEntity(
dataset=None,
data_dict=data_dict,
root_schema_class=schema.StateIncarcerationSentence,
root_entity_class=entities.StateIncarcerationSentence,
unifying_id_field='person_id',
build_related_entities=True))
# Get StateSupervisionSentences
supervision_sentences = (
test_pipeline | 'Load StateSupervisionSentences' >>
extractor_utils.BuildRootEntity(
dataset=None,
data_dict=data_dict,
root_schema_class=schema.StateSupervisionSentence,
root_entity_class=entities.StateSupervisionSentence,
unifying_id_field='person_id',
build_related_entities=True))
sentences_and_sentence_groups = (
{
'sentence_groups': sentence_groups,
'incarceration_sentences': incarceration_sentences,
'supervision_sentences': supervision_sentences
}
| 'Group sentences to sentence groups' >> beam.CoGroupByKey())
sentence_groups_with_hydrated_sentences = (
sentences_and_sentence_groups
| 'Set hydrated sentences on sentence groups' >> beam.ParDo(
SetSentencesOnSentenceGroup()))
# Group each StatePerson with their related entities
person_and_sentence_groups = (
{
'person': persons,
'sentence_groups': sentence_groups_with_hydrated_sentences
}
| 'Group StatePerson to SentenceGroups' >> beam.CoGroupByKey())
# Identify IncarcerationEvents events from the StatePerson's
# StateIncarcerationPeriods
fake_person_id_to_county_query_result = [{
'person_id':
fake_person_id,
'county_of_residence':
_COUNTY_OF_RESIDENCE
}]
person_id_to_county_kv = (
test_pipeline
| "Read person id to county associations from BigQuery" >>
beam.Create(fake_person_id_to_county_query_result)
|
"Convert to KV" >> beam.ParDo(ConvertDictToKVTuple(), 'person_id'))
person_events = (person_and_sentence_groups
| 'Classify Incarceration Events' >> beam.ParDo(
pipeline.ClassifyIncarcerationEvents(),
AsDict(person_id_to_county_kv)))
# Get pipeline job details for accessing job_id
all_pipeline_options = PipelineOptions().get_all_options()
# Add timestamp for local jobs
job_timestamp = datetime.datetime.now().strftime(
'%Y-%m-%d_%H_%M_%S.%f')
all_pipeline_options['job_timestamp'] = job_timestamp
# Get IncarcerationMetrics
incarceration_metrics = (
person_events
| 'Get Incarceration Metrics' >> pipeline.GetIncarcerationMetrics(
pipeline_options=all_pipeline_options,
inclusions=ALL_INCLUSIONS_DICT,
calculation_month_limit=-1))
assert_that(incarceration_metrics,
AssertMatchers.validate_metric_type())
test_pipeline.run()
def test_mergeIncarcerationPeriods_multipleTransfersSameDate(self) -> None:
incarceration_period_1 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
)
incarceration_period_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
incarceration_period_3 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_3,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY_2,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
)
incarceration_period_4 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_4,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_2,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
incarceration_period_5 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_5,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY_3,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
)
incarceration_period_6 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_6,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_3,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
)
expected_merged_incarceration_period_1 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID + "|" + _EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
expected_merged_incarceration_period_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_3 + "|" + _EXTERNAL_ID_4,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_2,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
expected_merged_incarceration_period_3 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_5 + "|" + _EXTERNAL_ID_6,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_3,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
)
expected_incarceration_periods = [
expected_merged_incarceration_period_1,
expected_merged_incarceration_period_2,
expected_merged_incarceration_period_3,
]
ingested_incarceration_periods = [
incarceration_period_1,
incarceration_period_5,
incarceration_period_2,
incarceration_period_4,
incarceration_period_3,
incarceration_period_6,
]
merged_periods = _merge_incarceration_periods_helper(
ingested_incarceration_periods)
self.assert_schema_object_lists_equal(expected_incarceration_periods,
merged_periods)
def test_mergeIncarcerationPeriods(self) -> None:
incarceration_period_1 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
)
incarceration_period_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
incarceration_period_3 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_3,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY_2,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
)
incarceration_period_4 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_4,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_2,
release_date=_DATE_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
incarceration_period_5 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_5,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_4,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
)
incarceration_period_6 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_6,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_3,
release_date=_DATE_5,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
placeholder_period = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
expected_merged_incarceration_period_1 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID + "|" + _EXTERNAL_ID_2,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
expected_merged_incarceration_period_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
external_id=_EXTERNAL_ID_3 + "|" + _EXTERNAL_ID_4,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
facility=_FACILITY_2,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER,
release_date=_DATE_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER,
)
expected_incarceration_periods = [
placeholder_period,
expected_merged_incarceration_period_1,
expected_merged_incarceration_period_2,
incarceration_period_5,
incarceration_period_6,
]
ingested_incarceration_periods = [
placeholder_period,
incarceration_period_1,
incarceration_period_5,
incarceration_period_2,
incarceration_period_4,
incarceration_period_3,
incarceration_period_6,
]
merged_periods = _merge_incarceration_periods_helper(
ingested_incarceration_periods)
self.assert_schema_object_lists_equal(expected_incarceration_periods,
merged_periods)
def test_transformToHolds_takeAdmissionReasonFromConsecutive(self) -> None:
# Arrange
# Too long of a time gap between date_1 and date_2 to be
# considered consecutive
date_1 = _DATE_1
date_2 = date_1 + datetime.timedelta(days=3)
date_3 = date_2 + datetime.timedelta(days=2)
ip = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=date_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION.value,
admission_reason_raw_text="PV",
release_date=date_1,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.EXTERNAL_UNKNOWN.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ip_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=date_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION.value,
release_date=date_2,
admission_reason_raw_text="ADM",
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ip_3 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=date_3,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER.
value,
admission_reason_raw_text="INT",
release_date=date_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.STATE_PRISON.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
expected_ip = attr.evolve(
self.to_entity(ip),
admission_reason=StateIncarcerationPeriodAdmissionReason.
TEMPORARY_CUSTODY,
release_reason=StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY,
)
expected_ip_2 = attr.evolve(
self.to_entity(ip_2),
admission_reason=StateIncarcerationPeriodAdmissionReason.
TEMPORARY_CUSTODY,
release_reason=StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY,
)
expected_ip_3 = attr.evolve(
self.to_entity(ip_3),
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
)
ips = [ip_2, ip, ip_3]
expected_ips = [expected_ip, expected_ip_2, expected_ip_3]
# Act
_update_temporary_holds_helper(ips, self.overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def test_transformToHolds(self) -> None:
# Arrange
ip = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=_DATE_1,
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION.value,
admission_reason_raw_text="PV",
release_date=_DATE_2,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.EXTERNAL_UNKNOWN.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ip_2 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=_DATE_2,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER.
value,
admission_reason_raw_text="INT",
release_date=_DATE_3,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ip_3 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=_DATE_3,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER.
value,
admission_reason_raw_text="INT",
release_date=_DATE_4,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.STATE_PRISON.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
ip_4 = schema.StateIncarcerationPeriod(
state_code=_STATE_CODE,
admission_date=_DATE_4,
admission_reason=StateIncarcerationPeriodAdmissionReason.TRANSFER.
value,
admission_reason_raw_text="INT",
release_date=_DATE_5,
release_reason=StateIncarcerationPeriodReleaseReason.TRANSFER.
value,
incarceration_type=StateIncarcerationType.COUNTY_JAIL.value,
status=StateIncarcerationPeriodStatus.PRESENT_WITHOUT_INFO,
)
expected_ip = attr.evolve(
self.to_entity(ip),
admission_reason=StateIncarcerationPeriodAdmissionReason.
TEMPORARY_CUSTODY,
release_reason=StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY,
)
expected_ip_2 = attr.evolve(
self.to_entity(ip_2),
admission_reason=StateIncarcerationPeriodAdmissionReason.
TEMPORARY_CUSTODY,
release_reason=StateIncarcerationPeriodReleaseReason.
RELEASED_FROM_TEMPORARY_CUSTODY,
)
expected_ip_3 = attr.evolve(
self.to_entity(ip_3),
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION,
)
expected_ip_4 = attr.evolve(self.to_entity(ip_4))
ips = [ip_2, ip_4, ip, ip_3]
expected_ips = [
expected_ip, expected_ip_2, expected_ip_3, expected_ip_4
]
# Act
_update_temporary_holds_helper(ips, self.overrides)
# Assert
entity_ips = [self.to_entity(ip) for ip in ips]
self.assertCountEqual(entity_ips, expected_ips)
def build_incarceration_pipeline_data_dict(self,
fake_person_id: int,
state_code: str = 'US_XX'):
"""Builds a data_dict for a basic run of the pipeline."""
fake_person = schema.StatePerson(
state_code=state_code,
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
persons_data = [normalized_database_base_dict(fake_person)]
race_1 = schema.StatePersonRace(person_race_id=111,
state_code=state_code,
race=Race.BLACK,
person_id=fake_person_id)
race_2 = schema.StatePersonRace(person_race_id=111,
state_code=state_code,
race=Race.WHITE,
person_id=fake_person_id)
races_data = normalized_database_base_dict_list([race_1, race_2])
ethnicity = schema.StatePersonEthnicity(person_ethnicity_id=111,
state_code=state_code,
ethnicity=Ethnicity.HISPANIC,
person_id=fake_person_id)
ethnicity_data = normalized_database_base_dict_list([ethnicity])
sentence_group = schema.StateSentenceGroup(sentence_group_id=111,
person_id=fake_person_id)
initial_incarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=1111,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code=state_code,
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2008, 11, 20),
release_date=date(2010, 12, 4),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id,
)
first_reincarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=2222,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code=state_code,
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2011, 4, 5),
release_date=date(2014, 4, 14),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id)
subsequent_reincarceration = schema.StateIncarcerationPeriod(
incarceration_period_id=3333,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.IN_CUSTODY,
state_code=state_code,
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2017, 1, 4),
person_id=fake_person_id)
incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_sentence_id=1111,
state_code=state_code,
sentence_group_id=sentence_group.sentence_group_id,
incarceration_periods=[
initial_incarceration, first_reincarceration,
subsequent_reincarceration
],
person_id=fake_person_id)
supervision_sentence = schema.StateSupervisionSentence(
supervision_sentence_id=123,
state_code=state_code,
person_id=fake_person_id)
sentence_group.incarceration_sentences = [incarceration_sentence]
sentence_group_data = [normalized_database_base_dict(sentence_group)]
incarceration_sentence_data = [
normalized_database_base_dict(incarceration_sentence)
]
supervision_sentence_data = [
normalized_database_base_dict(supervision_sentence)
]
incarceration_periods_data = [
normalized_database_base_dict(initial_incarceration),
normalized_database_base_dict(first_reincarceration),
normalized_database_base_dict(subsequent_reincarceration)
]
state_incarceration_sentence_incarceration_period_association = [
{
'incarceration_period_id':
initial_incarceration.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
{
'incarceration_period_id':
first_reincarceration.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
{
'incarceration_period_id':
subsequent_reincarceration.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
]
data_dict = self._default_data_dict()
data_dict_overrides = {
schema.StatePerson.__tablename__:
persons_data,
schema.StatePersonRace.__tablename__:
races_data,
schema.StatePersonEthnicity.__tablename__:
ethnicity_data,
schema.StateSentenceGroup.__tablename__:
sentence_group_data,
schema.StateIncarcerationSentence.__tablename__:
incarceration_sentence_data,
schema.StateSupervisionSentence.__tablename__:
supervision_sentence_data,
schema.StateIncarcerationPeriod.__tablename__:
incarceration_periods_data,
schema.state_incarceration_sentence_incarceration_period_association_table.name:
state_incarceration_sentence_incarceration_period_association,
}
data_dict.update(data_dict_overrides)
return data_dict
def build_incarceration_pipeline_data_dict_no_incarceration(
self, fake_person_id: int):
"""Builds a data_dict for a run of the pipeline where the person has no incarceration."""
fake_person_1 = schema.StatePerson(
state_code='US_XX',
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
fake_person_id_2 = 6789
fake_person_2 = schema.StatePerson(
state_code='US_XX',
person_id=fake_person_id_2,
gender=Gender.FEMALE,
birthdate=date(1990, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
persons_data = [
normalized_database_base_dict(fake_person_1),
normalized_database_base_dict(fake_person_2)
]
sentence_group = schema.StateSentenceGroup(sentence_group_id=111,
person_id=fake_person_id)
incarceration_period = schema.StateIncarcerationPeriod(
incarceration_period_id=1111,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code='US_XX',
county_code='124',
facility='San Quentin',
facility_security_level=StateIncarcerationFacilitySecurityLevel.
MAXIMUM,
admission_reason=StateIncarcerationPeriodAdmissionReason.
NEW_ADMISSION,
projected_release_reason=StateIncarcerationPeriodReleaseReason.
CONDITIONAL_RELEASE,
admission_date=date(2008, 11, 20),
release_date=date(2010, 12, 4),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
person_id=fake_person_id)
incarceration_sentence = schema.StateIncarcerationSentence(
incarceration_sentence_id=1111,
sentence_group_id=sentence_group.sentence_group_id,
incarceration_periods=[incarceration_period],
person_id=fake_person_id)
supervision_sentence = schema.StateSupervisionSentence(
supervision_sentence_id=123, person_id=fake_person_id)
sentence_group.incarceration_sentences = [incarceration_sentence]
sentence_group_data = [normalized_database_base_dict(sentence_group)]
incarceration_sentence_data = [
normalized_database_base_dict(incarceration_sentence)
]
supervision_sentence_data = [
normalized_database_base_dict(supervision_sentence)
]
incarceration_periods_data = [
normalized_database_base_dict(incarceration_period)
]
state_incarceration_sentence_incarceration_period_association = [
{
'incarceration_period_id':
incarceration_period.incarceration_period_id,
'incarceration_sentence_id':
incarceration_sentence.incarceration_sentence_id,
},
]
data_dict = self._default_data_dict()
data_dict_overrides = {
schema.StatePerson.__tablename__:
persons_data,
schema.StateSentenceGroup.__tablename__:
sentence_group_data,
schema.StateIncarcerationSentence.__tablename__:
incarceration_sentence_data,
schema.StateSupervisionSentence.__tablename__:
supervision_sentence_data,
schema.StateIncarcerationPeriod.__tablename__:
incarceration_periods_data,
schema.state_incarceration_sentence_incarceration_period_association_table.name:
state_incarceration_sentence_incarceration_period_association,
}
data_dict.update(data_dict_overrides)
return data_dict