def test_map_incarceration_combinations_two_releases_same_month(self):
person = StatePerson.new_with_defaults(
state_code="CA",
person_id=12345,
birthdate=date(1984, 8, 31),
gender=Gender.FEMALE,
)
race = StatePersonRace.new_with_defaults(state_code="CA",
race=Race.WHITE)
person.races = [race]
ethnicity = StatePersonEthnicity.new_with_defaults(
state_code="CA", ethnicity=Ethnicity.NOT_HISPANIC)
person.ethnicities = [ethnicity]
incarceration_events = [
IncarcerationReleaseEvent(
state_code="CA",
event_date=date(2010, 3, 12),
facility="FACILITY 33",
county_of_residence=_COUNTY_OF_RESIDENCE,
),
IncarcerationReleaseEvent(
state_code="CA",
event_date=date(2010, 3, 24),
facility="FACILITY 33",
county_of_residence=_COUNTY_OF_RESIDENCE,
),
]
incarceration_combinations = calculator.map_incarceration_combinations(
person=person,
incarceration_events=incarceration_events,
metric_inclusions=ALL_METRICS_INCLUSIONS_DICT,
calculation_end_month=None,
calculation_month_count=-1,
person_metadata=_DEFAULT_PERSON_METADATA,
)
expected_combinations_count = expected_metric_combos_count(
incarceration_events)
self.assertEqual(expected_combinations_count,
len(incarceration_combinations))
assert all(value == 1
for _combination, value in incarceration_combinations)
def test_map_incarceration_combinations_all_types(self):
person = StatePerson.new_with_defaults(person_id=12345,
birthdate=date(1984, 8, 31),
gender=Gender.FEMALE)
race = StatePersonRace.new_with_defaults(state_code='CA',
race=Race.WHITE)
person.races = [race]
ethnicity = StatePersonEthnicity.new_with_defaults(
state_code='CA', ethnicity=Ethnicity.NOT_HISPANIC)
person.ethnicities = [ethnicity]
incarceration_events = [
IncarcerationStayEvent(
admission_reason=StateIncarcerationPeriodAdmissionReason.
PAROLE_REVOCATION,
admission_reason_raw_text='NEW_ADMISSION',
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PAROLE,
state_code='CA',
event_date=date(2000, 3, 31),
facility='SAN QUENTIN',
county_of_residence=_COUNTY_OF_RESIDENCE,
most_serious_offense_statute=_STATUTE,
),
IncarcerationAdmissionEvent(
state_code='CA',
event_date=date(2000, 3, 12),
facility='SAN QUENTIN',
county_of_residence=_COUNTY_OF_RESIDENCE,
admission_reason=AdmissionReason.PAROLE_REVOCATION,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PAROLE,
admission_reason_raw_text='PAROLE_REVOCATION',
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.TREATMENT_IN_PRISON),
IncarcerationReleaseEvent(
state_code='CA',
event_date=date(2003, 4, 12),
facility='SAN QUENTIN',
county_of_residence=_COUNTY_OF_RESIDENCE,
)
]
incarceration_combinations = calculator.map_incarceration_combinations(
person=person,
incarceration_events=incarceration_events,
inclusions=ALL_INCLUSIONS_DICT,
calculation_month_limit=-1)
expected_combinations_count = expected_metric_combos_count(
person, incarceration_events, ALL_INCLUSIONS_DICT)
self.assertEqual(expected_combinations_count,
len(incarceration_combinations))
assert all(value == 1
for _combination, value in incarceration_combinations)
def release_event_for_period(
incarceration_sentences: List[StateIncarcerationSentence],
supervision_sentences: List[StateSupervisionSentence],
incarceration_period: StateIncarcerationPeriod,
county_of_residence: Optional[str]
) -> Optional[IncarcerationReleaseEvent]:
"""Returns an IncarcerationReleaseEvent if this incarceration period represents an release from incarceration."""
release_date = incarceration_period.release_date
release_reason = incarceration_period.release_reason
purpose_for_incarceration = incarceration_period.specialized_purpose_for_incarceration
if release_date and release_reason:
supervision_type_at_release = get_post_incarceration_supervision_type(
incarceration_sentences, supervision_sentences,
incarceration_period)
return IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=release_date,
facility=incarceration_period.facility,
release_reason=release_reason,
release_reason_raw_text=incarceration_period.
release_reason_raw_text,
purpose_for_incarceration=purpose_for_incarceration,
county_of_residence=county_of_residence,
supervision_type_at_release=supervision_type_at_release)
return None
def release_event_for_period(
incarceration_sentences: List[StateIncarcerationSentence],
supervision_sentences: List[StateSupervisionSentence],
incarceration_period: StateIncarcerationPeriod,
county_of_residence: Optional[str],
) -> Optional[IncarcerationReleaseEvent]:
"""Returns an IncarcerationReleaseEvent if this incarceration period represents an release from incarceration."""
release_date = incarceration_period.release_date
release_reason = incarceration_period.release_reason
purpose_for_incarceration = (
incarceration_period.specialized_purpose_for_incarceration)
supervision_type_at_admission = get_pre_incarceration_supervision_type(
incarceration_sentences, supervision_sentences, incarceration_period)
admission_reason = None
if incarceration_period.admission_reason:
admission_reason = state_specific_incarceration_admission_reason_override(
incarceration_period.state_code,
incarceration_period.admission_reason,
supervision_type_at_admission,
)
if release_date and release_reason:
supervision_type_at_release = get_post_incarceration_supervision_type(
incarceration_sentences, supervision_sentences,
incarceration_period)
total_days_incarcerated = None
if incarceration_period.admission_date:
total_days_incarcerated = (
release_date - incarceration_period.admission_date).days
if total_days_incarcerated < 0:
logging.warning(
"release_date before admission_date on incarceration period: %s",
incarceration_period,
)
total_days_incarcerated = 0
return IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=release_date,
facility=incarceration_period.facility,
release_reason=release_reason,
release_reason_raw_text=incarceration_period.
release_reason_raw_text,
purpose_for_incarceration=purpose_for_incarceration,
county_of_residence=county_of_residence,
supervision_type_at_release=supervision_type_at_release,
admission_reason=admission_reason,
total_days_incarcerated=total_days_incarcerated,
)
return None
def testCalculateIncarcerationMetricCombinations(self):
"""Tests the CalculateIncarcerationMetricCombinations DoFn."""
fake_person = StatePerson.new_with_defaults(
person_id=123,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
incarceration_events = [
IncarcerationAdmissionEvent(
state_code='US_XX',
event_date=date(2001, 3, 16),
facility='SAN QUENTIN',
county_of_residence='county_of_residence',
admission_reason=StateIncarcerationPeriodAdmissionReason.
PROBATION_REVOCATION),
IncarcerationReleaseEvent(
state_code='US_XX',
event_date=date(2002, 5, 26),
facility='SAN QUENTIN',
county_of_residence='county_of_residence',
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED)
]
# One metric per methodology for each event
expected_metric_count = 2
expected_combination_counts = \
{'admissions': expected_metric_count,
'releases': expected_metric_count}
test_pipeline = TestPipeline()
inputs = [(self.fake_person_id, {
'person_events': [(fake_person, incarceration_events)],
'person_metadata': [self.person_metadata]
})]
output = (test_pipeline
| beam.Create(inputs)
| beam.ParDo(ExtractPersonEventsMetadata())
| 'Calculate Incarceration Metrics' >> beam.ParDo(
pipeline.CalculateIncarcerationMetricCombinations(),
None, -1, ALL_METRICS_INCLUSIONS_DICT))
assert_that(
output,
AssertMatchers.count_combinations(expected_combination_counts),
'Assert number of metrics is expected value')
test_pipeline.run()
def test_map_incarceration_combinations_relevant_periods(self):
person = StatePerson.new_with_defaults(person_id=12345,
birthdate=date(1984, 8, 31),
gender=Gender.FEMALE)
race = StatePersonRace.new_with_defaults(state_code='CA',
race=Race.WHITE)
person.races = [race]
ethnicity = StatePersonEthnicity.new_with_defaults(
state_code='CA', ethnicity=Ethnicity.NOT_HISPANIC)
person.ethnicities = [ethnicity]
incarceration_events = [
IncarcerationAdmissionEvent(
state_code='CA',
event_date=date(2010, 10, 2),
facility='SAN QUENTIN',
county_of_residence=_COUNTY_OF_RESIDENCE,
),
IncarcerationReleaseEvent(
state_code='CA',
event_date=date(2010, 10, 19),
facility='SAN QUENTIN',
county_of_residence=_COUNTY_OF_RESIDENCE,
)
]
incarceration_combinations = calculator.map_incarceration_combinations(
person=person,
incarceration_events=incarceration_events,
inclusions=ALL_INCLUSIONS_DICT,
calculation_month_limit=-1)
expected_combinations_count = expected_metric_combos_count(
person, incarceration_events, ALL_INCLUSIONS_DICT,
len(calculator_utils.METRIC_PERIOD_MONTHS),
len(calculator_utils.METRIC_PERIOD_MONTHS))
self.assertEqual(expected_combinations_count,
len(incarceration_combinations))
assert all(value == 1
for _combination, value in incarceration_combinations)
for combo, _ in incarceration_combinations:
assert combo.get('year') == 2010
def release_event_for_period(
incarceration_period: StateIncarcerationPeriod,
county_of_residence: Optional[str]) -> Optional[IncarcerationReleaseEvent]:
"""Returns an IncarcerationReleaseEvent if this incarceration period represents an release from incarceration."""
release_date = incarceration_period.release_date
release_reason = incarceration_period.release_reason
incarceration_type = incarceration_period.incarceration_type
if release_date and release_reason and \
incarceration_type == StateIncarcerationType.STATE_PRISON:
return IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=release_date,
facility=incarceration_period.facility,
release_reason=release_reason,
county_of_residence=county_of_residence,
)
return None
def release_event_for_period(
incarceration_period: StateIncarcerationPeriod,
county_of_residence: Optional[str]
) -> Optional[IncarcerationReleaseEvent]:
"""Returns an IncarcerationReleaseEvent if this incarceration period represents an release from incarceration."""
release_date = incarceration_period.release_date
release_reason = incarceration_period.release_reason
purpose_for_incarceration = incarceration_period.specialized_purpose_for_incarceration
if release_date and release_reason:
return IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=release_date,
facility=incarceration_period.facility,
release_reason=release_reason,
release_reason_raw_text=incarceration_period.
release_reason_raw_text,
purpose_for_incarceration=purpose_for_incarceration,
county_of_residence=county_of_residence,
)
return None
def testClassifyIncarcerationEvents(self):
"""Tests the ClassifyIncarcerationEvents DoFn."""
fake_person_id = 12345
fake_person = StatePerson.new_with_defaults(
state_code='US_XX',
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
incarceration_period = StateIncarcerationPeriod.new_with_defaults(
incarceration_period_id=1111,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code='TX',
facility='PRISON XX',
admission_date=date(2010, 11, 20),
admission_reason=StateIncarcerationPeriodAdmissionReason.
PROBATION_REVOCATION,
release_date=date(2010, 11, 21),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD)
incarceration_sentence = StateIncarcerationSentence.new_with_defaults(
incarceration_sentence_id=123,
incarceration_periods=[incarceration_period],
start_date=date(2009, 2, 9),
charges=[
StateCharge.new_with_defaults(ncic_code='5699',
statute='30A123',
offense_date=date(2009, 1, 9))
])
sentence_group = StateSentenceGroup.new_with_defaults(
sentence_group_id=123,
incarceration_sentences=[incarceration_sentence])
incarceration_sentence.sentence_group = sentence_group
incarceration_period.incarceration_sentences = [incarceration_sentence]
fake_person_id_to_county_query_result = [{
'person_id':
fake_person_id,
'county_of_residence':
_COUNTY_OF_RESIDENCE
}]
fake_incarceration_period_judicial_district_association_result = \
{'person_id': fake_person_id, 'incarceration_period_id': 123, 'judicial_district_code': 'NW'}
incarceration_events = [
IncarcerationStayEvent(
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.
admission_reason_raw_text,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PROBATION,
state_code=incarceration_period.state_code,
event_date=incarceration_period.admission_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
most_serious_offense_ncic_code='5699',
most_serious_offense_statute='30A123',
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD),
IncarcerationAdmissionEvent(
state_code=incarceration_period.state_code,
event_date=incarceration_period.admission_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.
admission_reason_raw_text,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PROBATION,
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD),
IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=incarceration_period.release_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
release_reason=incarceration_period.release_reason,
admission_reason=incarceration_period.admission_reason,
total_days_incarcerated=(
incarceration_period.release_date -
incarceration_period.admission_date).days,
purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD)
]
correct_output = [(fake_person_id, (fake_person, incarceration_events))
]
test_pipeline = TestPipeline()
person_id_to_county_kv = (
test_pipeline
| "Read person id to county associations from BigQuery" >>
beam.Create(fake_person_id_to_county_query_result)
|
"Convert to KV" >> beam.ParDo(ConvertDictToKVTuple(), 'person_id'))
person_entities = {
'person': [fake_person],
'sentence_groups': [sentence_group],
'incarceration_period_judicial_district_association':
[fake_incarceration_period_judicial_district_association_result]
}
output = (test_pipeline
| beam.Create([(fake_person_id, person_entities)])
| 'Identify Incarceration Events' >> beam.ParDo(
pipeline.ClassifyIncarcerationEvents(),
AsDict(person_id_to_county_kv)))
assert_that(output, equal_to(correct_output))
test_pipeline.run()
def testCalculateIncarcerationMetricCombinations(self):
"""Tests the CalculateIncarcerationMetricCombinations DoFn."""
fake_person = StatePerson.new_with_defaults(
person_id=123,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
incarceration_events = [
IncarcerationAdmissionEvent(
state_code='CA',
event_date=date(2001, 3, 16),
facility='SAN QUENTIN',
county_of_residence='county_of_residence',
admission_reason=StateIncarcerationPeriodAdmissionReason.
PROBATION_REVOCATION),
IncarcerationReleaseEvent(
state_code='CA',
event_date=date(2002, 5, 26),
facility='SAN QUENTIN',
county_of_residence='county_of_residence',
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED)
]
# Get the number of combinations of person-event characteristics.
num_combinations = len(
calculator.characteristic_combinations(
fake_person, incarceration_events[0], ALL_INCLUSIONS_DICT,
IncarcerationMetricType.POPULATION))
assert num_combinations > 0
expected_metric_count = num_combinations * 2
expected_admission_combination_counts = \
{'admissions': expected_metric_count}
expected_releases_combination_counts = \
{'releases': expected_metric_count}
test_pipeline = TestPipeline()
output = (
test_pipeline
| beam.Create([(fake_person, incarceration_events)])
| 'Calculate Incarceration Metrics' >> beam.ParDo(
pipeline.CalculateIncarcerationMetricCombinations(), -1,
ALL_INCLUSIONS_DICT).with_outputs('admissions', 'releases'))
assert_that(
output.admissions,
AssertMatchers.count_combinations(
expected_admission_combination_counts),
'Assert number of admission metrics is expected value')
assert_that(
output.releases,
AssertMatchers.count_combinations(
expected_releases_combination_counts),
'Assert number of release metrics is expected value')
test_pipeline.run()
def testCalculateIncarcerationMetricCombinations(self):
"""Tests the CalculateIncarcerationMetricCombinations DoFn."""
fake_person = StatePerson.new_with_defaults(
state_code="US_XX",
person_id=123,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT,
)
incarceration_events = [
IncarcerationAdmissionEvent(
state_code="US_XX",
event_date=date(2001, 3, 16),
facility="SAN QUENTIN",
county_of_residence="county_of_residence",
admission_reason=StateIncarcerationPeriodAdmissionReason.
PROBATION_REVOCATION,
),
IncarcerationReleaseEvent(
state_code="US_XX",
event_date=date(2002, 5, 26),
facility="SAN QUENTIN",
county_of_residence="county_of_residence",
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
),
]
expected_metric_count = 1
expected_combination_counts = {
"admissions": expected_metric_count,
"releases": expected_metric_count,
}
test_pipeline = TestPipeline()
inputs = [(
self.fake_person_id,
{
"person_events": [(fake_person, incarceration_events)],
"person_metadata": [self.person_metadata],
},
)]
output = (test_pipeline
| beam.Create(inputs)
| beam.ParDo(ExtractPersonEventsMetadata())
| "Calculate Incarceration Metrics" >> beam.ParDo(
pipeline.CalculateIncarcerationMetricCombinations(),
None,
-1,
ALL_METRICS_INCLUSIONS_DICT,
))
assert_that(
output,
AssertMatchers.count_combinations(expected_combination_counts),
"Assert number of metrics is expected value",
)
test_pipeline.run()
def testClassifyIncarcerationEvents(self):
"""Tests the ClassifyIncarcerationEvents DoFn."""
fake_person_id = 12345
fake_person = StatePerson.new_with_defaults(
state_code="US_XX",
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT,
)
incarceration_period = StateIncarcerationPeriod.new_with_defaults(
incarceration_period_id=1111,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code="TX",
facility="PRISON XX",
admission_date=date(2010, 11, 20),
admission_reason=StateIncarcerationPeriodAdmissionReason.
PROBATION_REVOCATION,
release_date=date(2010, 11, 21),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED,
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD,
)
incarceration_sentence = StateIncarcerationSentence.new_with_defaults(
state_code="US_XX",
incarceration_sentence_id=123,
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
incarceration_periods=[incarceration_period],
start_date=date(2009, 2, 9),
charges=[
StateCharge.new_with_defaults(
state_code="US_XX",
status=ChargeStatus.PRESENT_WITHOUT_INFO,
ncic_code="5699",
statute="30A123",
offense_date=date(2009, 1, 9),
)
],
)
sentence_group = StateSentenceGroup.new_with_defaults(
state_code="US_XX",
status=StateSentenceStatus.PRESENT_WITHOUT_INFO,
sentence_group_id=123,
incarceration_sentences=[incarceration_sentence],
)
incarceration_sentence.sentence_group = sentence_group
incarceration_period.incarceration_sentences = [incarceration_sentence]
fake_person_id_to_county_query_result = {
"person_id": fake_person_id,
"county_of_residence": _COUNTY_OF_RESIDENCE,
}
fake_incarceration_period_judicial_district_association_result = {
"person_id": fake_person_id,
"incarceration_period_id": 123,
"judicial_district_code": "NW",
}
incarceration_events = [
IncarcerationStayEvent(
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.
admission_reason_raw_text,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PROBATION,
state_code=incarceration_period.state_code,
event_date=incarceration_period.admission_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
most_serious_offense_ncic_code="5699",
most_serious_offense_statute="30A123",
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD,
),
IncarcerationAdmissionEvent(
state_code=incarceration_period.state_code,
event_date=incarceration_period.admission_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.
admission_reason_raw_text,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PROBATION,
specialized_purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD,
),
IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=incarceration_period.release_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
release_reason=incarceration_period.release_reason,
admission_reason=incarceration_period.admission_reason,
total_days_incarcerated=(
incarceration_period.release_date -
incarceration_period.admission_date).days,
purpose_for_incarceration=
StateSpecializedPurposeForIncarceration.PAROLE_BOARD_HOLD,
),
]
correct_output = [(fake_person_id, (fake_person, incarceration_events))
]
test_pipeline = TestPipeline()
person_entities = {
"person": [fake_person],
"sentence_groups": [sentence_group],
"incarceration_period_judicial_district_association":
[fake_incarceration_period_judicial_district_association_result],
"persons_to_recent_county_of_residence":
[fake_person_id_to_county_query_result],
}
output = (test_pipeline
| beam.Create([(fake_person_id, person_entities)])
| "Identify Incarceration Events" >> beam.ParDo(
pipeline.ClassifyIncarcerationEvents()))
assert_that(output, equal_to(correct_output))
test_pipeline.run()