def month_is_non_revocation_supervision_bucket(
start_of_month: date, termination_date: date,
months_fully_incarcerated: Set[Tuple[int, int]],
incarceration_periods_by_admission_month: Dict[int, Dict[
int, List[StateIncarcerationPeriod]]]):
"""Determines whether the given month was a month on supervision without a revocation and without being
incarcerated for the full time spent on supervision that month."""
was_incarcerated_all_month = (
start_of_month.year, start_of_month.month) in months_fully_incarcerated
if was_incarcerated_all_month:
return False
if has_revocation_admission_in_month(
start_of_month, incarceration_periods_by_admission_month):
return False
if last_day_of_month(termination_date) == last_day_of_month(
start_of_month):
# If the supervision period ended this month, make sure there wasn't an incarceration period that
# fully overlapped with the days of supervision in this month
incarceration_overlapping_supervision_this_month = [
ip for months in incarceration_periods_by_admission_month.values()
for ips_in_month in months.values() for ip in ips_in_month
if ip.admission_date and ip.admission_date <= start_of_month and (
ip.release_date is None or termination_date <= ip.release_date)
]
if incarceration_overlapping_supervision_this_month:
return False
return True
def find_end_of_month_state_prison_stays(
incarceration_sentences: List[StateIncarcerationSentence],
supervision_sentences: List[StateSupervisionSentence],
incarceration_period: StateIncarcerationPeriod,
county_of_residence: Optional[str]) -> List[IncarcerationStayEvent]:
"""Finds months for which this person was incarcerated in a state prison on the last day of the month.
"""
incarceration_stay_events: List[IncarcerationStayEvent] = []
if incarceration_period.incarceration_type != StateIncarcerationType.STATE_PRISON:
return incarceration_stay_events
admission_date = incarceration_period.admission_date
release_date = incarceration_period.release_date
if release_date is None:
release_date = date.today()
if admission_date is None:
return incarceration_stay_events
supervision_type_at_admission = get_pre_incarceration_supervision_type(
incarceration_sentences, supervision_sentences, incarceration_period)
sentence_group = _get_sentence_group_for_incarceration_period(incarceration_period)
end_of_month = last_day_of_month(admission_date)
while end_of_month < release_date:
most_serious_charge = find_most_serious_prior_charge_in_sentence_group(sentence_group, end_of_month)
most_serious_offense_ncic_code = most_serious_charge.ncic_code if most_serious_charge else None
most_serious_offense_statute = most_serious_charge.statute if most_serious_charge else None
incarceration_stay_events.append(
IncarcerationStayEvent(
state_code=incarceration_period.state_code,
event_date=end_of_month,
facility=incarceration_period.facility,
county_of_residence=county_of_residence,
most_serious_offense_ncic_code=most_serious_offense_ncic_code,
most_serious_offense_statute=most_serious_offense_statute,
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.admission_reason_raw_text,
supervision_type_at_admission=supervision_type_at_admission,
)
)
end_of_month = last_day_of_month(end_of_month + relativedelta(days=1))
return incarceration_stay_events
def _identify_months_fully_incarcerated(
incarceration_periods: List[StateIncarcerationPeriod]) -> \
Set[Tuple[int, int]]:
"""For each StateIncarcerationPeriod, identifies months where the person was incarcerated for every day during that
month. Returns a set of months in the format (year, month) for which the person spent the entire month in a
prison."""
months_incarcerated: Set[Tuple[int, int]] = set()
for incarceration_period in incarceration_periods:
admission_date = incarceration_period.admission_date
release_date = incarceration_period.release_date
if admission_date is None:
return months_incarcerated
if release_date is None:
release_date = last_day_of_month(date.today())
if admission_date.day == 1:
month_date = admission_date
else:
month_date = first_day_of_month(admission_date) + relativedelta(
months=1)
while month_date + relativedelta(
months=1) <= release_date + relativedelta(days=1):
months_incarcerated.add((month_date.year, month_date.month))
month_date = month_date + relativedelta(months=1)
return months_incarcerated
def us_mo_get_month_supervision_type(
any_date_in_month: datetime.date,
supervision_sentences: List[StateSupervisionSentence],
incarceration_sentences: List[StateIncarcerationSentence],
supervision_period: StateSupervisionPeriod
) -> StateSupervisionPeriodSupervisionType:
"""Calculates the supervision period supervision type that should be attributed to a US_MO supervision period
on a given month.
The date used to calculate the supervision period supervision type is either the last day of the month, or
the last day of supervision, whichever comes first.
"""
end_of_month = last_day_of_month(any_date_in_month)
if supervision_period.termination_date is None:
supervision_type_determination_date = end_of_month
else:
supervision_type_determination_date = min(
end_of_month,
supervision_period.termination_date - datetime.timedelta(days=1))
supervision_type = \
us_mo_get_supervision_period_supervision_type_on_date(supervision_type_determination_date,
supervision_sentences,
incarceration_sentences)
if not supervision_type:
return StateSupervisionPeriodSupervisionType.INTERNAL_UNKNOWN
return supervision_type
def us_mo_get_month_supervision_type(
any_date_in_month: datetime.date,
supervision_sentences: List[StateSupervisionSentence],
incarceration_sentences: List[StateIncarcerationSentence],
supervision_period: StateSupervisionPeriod
) -> StateSupervisionPeriodSupervisionType:
"""Calculates the supervision period supervision type that should be attributed to a US_MO supervision period
on a given month.
The date used to calculate the supervision period supervision type is either the last day of the month, or
the last day of supervision, whichever comes first.
"""
start_of_month = first_day_of_month(any_date_in_month)
end_of_month = last_day_of_month(any_date_in_month)
first_of_next_month = end_of_month + datetime.timedelta(days=1)
if supervision_period.termination_date is None:
upper_bound_exclusive_date = first_of_next_month
else:
upper_bound_exclusive_date = min(first_of_next_month,
supervision_period.termination_date)
lower_bound_inclusive = max(start_of_month, supervision_period.start_date)
supervision_type = \
us_mo_get_most_recent_supervision_period_supervision_type_before_upper_bound_day(
upper_bound_exclusive_date=upper_bound_exclusive_date,
lower_bound_inclusive_date=lower_bound_inclusive,
supervision_sentences=supervision_sentences,
incarceration_sentences=incarceration_sentences)
if not supervision_type:
return StateSupervisionPeriodSupervisionType.INTERNAL_UNKNOWN
return supervision_type
def combination_referral_monthly_metrics(
combo: Dict[str, Any],
program_event: ProgramReferralEvent,
all_referral_events:
List[ProgramReferralEvent]) \
-> List[Tuple[Dict[str, Any], int]]:
"""Returns all unique referral metrics for the given event and combination.
First, includes an event-based count for the month the event occurred with a metric period of 1 month. Then, if
this event should be included in the person-based count for the month when the event occurred, adds those person-
based metrics.
Args:
combo: A characteristic combination to convert into metrics
program_event: The program event from which the combination was derived
all_referral_events: All of this person's ProgramReferralEvents
Returns:
A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing
a positive contribution to that count metric.
"""
metrics = []
event_date = program_event.event_date
event_year = event_date.year
event_month = event_date.month
# Add event-based combo for the 1-month period the month of the event
event_based_same_month_combo = augmented_combo_for_calculations(
combo, program_event.state_code,
event_year, event_month,
MetricMethodologyType.EVENT, 1)
metrics.append((event_based_same_month_combo, 1))
# Create the person-based combo for the 1-month period of the month of the event
person_based_same_month_combo = augmented_combo_for_calculations(
combo, program_event.state_code,
event_year, event_month,
MetricMethodologyType.PERSON, 1
)
# Get all other referral events that happened the same month as this one
all_referral_events_in_event_month = [
event for event in all_referral_events
if event.event_date.year == event_date.year and
event.event_date.month == event_date.month
]
if include_referral_in_count(
combo,
program_event,
last_day_of_month(event_date),
all_referral_events_in_event_month):
# Include this event in the person-based count
metrics.append((person_based_same_month_combo, 1))
return metrics
def map_recidivism_count_combinations(
characteristic_combos: List[Dict[str, Any]],
event: ReleaseEvent,
all_reincarcerations: Dict[date, Dict[str, Any]],
metric_period_end_date: date) -> \
List[Tuple[Dict[str, Any], Any]]:
"""Maps the given event and characteristic combinations to a variety of metrics that track count-based recidivism.
If the event is a RecidivismReleaseEvent, then a count of reincarceration occurred. This produces metrics for both
the year and the month in which the person was reincarcerated.
Args:
characteristic_combos: A list of dictionaries containing all unique combinations of characteristics.
event: the recidivism event from which the combination was derived
all_reincarcerations: dictionary where the keys are all dates of reincarceration for the person's ReleaseEvents,
and the values are a dictionary containing return type and from supervision type information
metric_period_end_date: The day the metric periods end
Returns:
A list of key-value tuples representing specific metric combinations and the recidivism value corresponding to
that metric.
"""
metrics = []
if isinstance(event, RecidivismReleaseEvent):
reincarceration_date = event.reincarceration_date
relevant_periods = relevant_metric_periods(
reincarceration_date, metric_period_end_date.year,
metric_period_end_date.month)
for combo in characteristic_combos:
combo['metric_type'] = ReincarcerationRecidivismMetricType.COUNT
# Bucket for the month of the incarceration
combo['year'] = reincarceration_date.year
combo['month'] = reincarceration_date.month
combo['metric_period_months'] = 1
end_of_event_month = last_day_of_month(reincarceration_date)
metrics.extend(
combination_count_metrics(combo, event, all_reincarcerations,
end_of_event_month))
# Bucket for each of the relevant metric period month lengths
for relevant_period in relevant_periods:
combo['year'] = metric_period_end_date.year
combo['month'] = metric_period_end_date.month
combo['metric_period_months'] = relevant_period
metrics.extend(
combination_count_metrics(combo, event,
all_reincarcerations,
metric_period_end_date))
return metrics
def map_recidivism_liberty_combinations(
characteristic_combos: List[Dict[str, Any]],
event: ReleaseEvent,
all_reincarcerations: Dict[date, Dict[str, Any]]) -> \
List[Tuple[Dict[str, Any], Any]]:
"""Maps the given event and characteristic combinations to a variety of metrics that track metrics for time at
liberty.
"""
metrics = []
if isinstance(event, RecidivismReleaseEvent):
reincarceration_date = event.reincarceration_date
year = reincarceration_date.year
year_start_day = date(year, 1, 1)
year_end_day = date(year, 12, 31)
month_start_day = first_day_of_month(reincarceration_date)
month_end_day = last_day_of_month(reincarceration_date)
for combo in characteristic_combos:
combo['metric_type'] = ReincarcerationRecidivismMetricType.LIBERTY
# Year bucket
combo['start_date'] = year_start_day
combo['end_date'] = year_end_day
metrics.extend(
combination_liberty_metrics(combo, event,
all_reincarcerations))
# Month bucket
combo['start_date'] = month_start_day
combo['end_date'] = month_end_day
metrics.extend(
combination_liberty_metrics(combo, event,
all_reincarcerations))
return metrics
def get_month_supervision_type(
any_date_in_month: datetime.date,
supervision_sentences: List[StateSupervisionSentence],
incarceration_sentences: List[StateIncarcerationSentence],
supervision_period: StateSupervisionPeriod
) -> StateSupervisionPeriodSupervisionType:
"""Supervision type can change over time even if the period does not change. This function calculates the
supervision type that a given supervision period represents during the month that |any_date_in_month| falls in. We
do this by looking at all sentences attached to this supervision period, then determining which ones overlap with
any day in the month, and using the sentence supervision types to determine the period supervision type at this
point in time.
Args:
any_date_in_month: (date) Any day in the month to consider
supervision_period: (StateSupervisionPeriod) The supervision period we want to associate a supervision type with
supervision_sentences: (List[StateSupervisionSentence]) All supervision sentences for a given person.
"""
if not supervision_period.supervision_period_id:
raise ValueError('All objects should have database ids.')
if is_placeholder(supervision_period):
raise ValueError('Do not expect placeholder periods!')
start_of_month = first_day_of_month(any_date_in_month)
end_of_month = last_day_of_month(any_date_in_month)
# Find sentences that are attached to the period and overlap with the month
incarceration_sentences = _get_valid_attached_sentences(
incarceration_sentences, supervision_period)
incarceration_sentences = _get_sentences_overlapping_with_dates(
start_of_month, end_of_month, incarceration_sentences)
supervision_sentences = _get_valid_attached_sentences(
supervision_sentences, supervision_period)
supervision_sentences = _get_sentences_overlapping_with_dates(
start_of_month, end_of_month, supervision_sentences)
return _get_supervision_type_from_sentences(incarceration_sentences,
supervision_sentences)
def map_incarceration_combinations(person: StatePerson,
incarceration_events:
List[IncarcerationEvent],
inclusions: Dict[str, bool],
calculation_month_limit: int) -> List[Tuple[Dict[str, Any], Any]]:
"""Transforms IncarcerationEvents and a StatePerson into metric combinations.
Takes in a StatePerson and all of their IncarcerationEvent and returns an array of "incarceration combinations".
These are key-value pairs where the key represents a specific metric and the value represents whether or not
the person should be counted as a positive instance of that metric.
This translates a particular incarceration event, e.g. admission or release, into many different incarceration
metrics. Each metric represents one of many possible combinations of characteristics being tracked for that event.
For example, if a White male is admitted to prison, there is a metric that corresponds to White people, one to
males, one to White males, one to all people, and more depending on other dimensions in the data.
Args:
person: the StatePerson
incarceration_events: A list of IncarcerationEvents for the given StatePerson.
inclusions: A dictionary containing the following keys that correspond to characteristic dimensions:
- age_bucket
- ethnicity
- gender
- race
Where the values are boolean flags indicating whether to include the dimension in the calculations.
calculation_month_limit: The number of months (including this one) to limit the monthly calculation output to.
If set to -1, does not limit the calculations.
Returns:
A list of key-value tuples representing specific metric combinations and the value corresponding to that metric.
"""
metrics: List[Tuple[Dict[str, Any], Any]] = []
periods_and_events: Dict[int, List[IncarcerationEvent]] = defaultdict()
# We will calculate person-based metrics for each metric period in METRIC_PERIOD_MONTHS ending with the current
# month
metric_period_end_date = last_day_of_month(date.today())
calculation_month_lower_bound = get_calculation_month_lower_bound_date(
metric_period_end_date, calculation_month_limit)
# Organize the events by the relevant metric periods
for incarceration_event in incarceration_events:
relevant_periods = relevant_metric_periods(
incarceration_event.event_date,
metric_period_end_date.year,
metric_period_end_date.month)
if relevant_periods:
for period in relevant_periods:
period_events = periods_and_events.get(period)
if period_events:
period_events.append(incarceration_event)
else:
periods_and_events[period] = [incarceration_event]
for incarceration_event in incarceration_events:
metric_type = METRIC_TYPES.get(type(incarceration_event))
if not metric_type:
raise ValueError(
'No metric type mapped to incarceration event of type {}'.format(type(incarceration_event)))
characteristic_combos = characteristic_combinations(person, incarceration_event, inclusions, metric_type)
metrics.extend(map_metric_combinations(
characteristic_combos, incarceration_event,
metric_period_end_date, calculation_month_lower_bound, incarceration_events,
periods_and_events, metric_type
))
return metrics
def combination_program_monthly_metrics(
combo: Dict[str, Any], program_event: ProgramEvent,
metric_type: ProgramMetricType, all_program_events: List[ProgramEvent],
is_daily_metric: bool) -> List[Tuple[Dict[str, Any], int]]:
"""Returns all unique referral metrics for the given event and combination.
First, includes an event-based count for the month the event occurred with a metric period of 1 month. Then, if
this event should be included in the person-based count for the month when the event occurred, adds those person-
based metrics.
Args:
combo: A characteristic combination to convert into metrics
program_event: The program event from which the combination was derived
metric_type: The type of metric being tracked by this combo
all_program_events: All of this person's ProgramEvents
is_daily_metric: If True, limits person-based counts to the date of the event. If False, limits person-based
counts to the month of the event.
Returns:
A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing
a positive contribution to that count metric.
"""
metrics = []
event_date = program_event.event_date
event_year = event_date.year
event_month = event_date.month
base_metric_period = 0 if is_daily_metric else 1
# Add event-based combo for the base metric period the month of the event
event_based_same_month_combo = augmented_combo_for_calculations(
combo, program_event.state_code, event_year, event_month,
MetricMethodologyType.EVENT, base_metric_period)
metrics.append((event_based_same_month_combo, 1))
# Create the person-based combo for the base metric period of the month of the event
person_based_same_month_combo = augmented_combo_for_calculations(
combo, program_event.state_code, event_year, event_month,
MetricMethodologyType.PERSON, base_metric_period)
events_in_period: List[ProgramEvent] = []
if metric_type == ProgramMetricType.PARTICIPATION:
# Get all other participation events that happened on the same day as this one
events_in_period = [
event for event in all_program_events
if (isinstance(event, ProgramParticipationEvent))
and event.event_date == program_event.event_date
]
elif metric_type == ProgramMetricType.REFERRAL:
# Get all other referral events that happened in the same month as this one
events_in_period = [
event for event in all_program_events
if isinstance(event, ProgramReferralEvent) and event.event_date.
year == event_year and event.event_date.month == event_month
]
if include_event_in_count(combo, program_event,
last_day_of_month(event_date), events_in_period):
# Include this event in the person-based count
metrics.append((person_based_same_month_combo, 1))
return metrics
def combination_incarceration_metrics(
combo: Dict[str, Any],
incarceration_event: IncarcerationEvent,
all_incarceration_events: List[IncarcerationEvent],
is_daily_metric: bool) \
-> List[Tuple[Dict[str, Any], int]]:
"""Returns all unique incarceration metrics for the given event and combination.
First, includes an event-based count for the event. Then, if this is a daily metric, includes a count of the event
if it should be included in the person-based count for the day when the event occurred. If this is not a daily
metric, includes a count of the event if it should be included in the person-based count for the month of the event.
Args:
combo: A characteristic combination to convert into metrics
incarceration_event: The IncarcerationEvent from which the combination was derived
all_incarceration_events: All of this person's IncarcerationEvents
is_daily_metric: If True, limits person-based counts to the date of the event. If False, limits person-based
counts to the month of the event.
Returns:
A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing
a positive contribution to that count metric.
"""
metrics = []
event_date = incarceration_event.event_date
event_year = event_date.year
event_month = event_date.month
metric_period_months = 0 if is_daily_metric else 1
# Add event-based combo for the 1-month period the month of the event
event_based_same_month_combo = augmented_combo_for_calculations(
combo,
incarceration_event.state_code,
event_year,
event_month,
MetricMethodologyType.EVENT,
metric_period_months=metric_period_months)
metrics.append((event_based_same_month_combo, 1))
# Create the person-based combo for the 1-month period of the month of the event
person_based_same_month_combo = augmented_combo_for_calculations(
combo,
incarceration_event.state_code,
event_year,
event_month,
MetricMethodologyType.PERSON,
metric_period_months=metric_period_months)
day_match_value = event_date.day if is_daily_metric else None
# Get the events of the same type that happened in the same month
events_in_period = matching_events_for_person_based_count(
year=event_year,
month=event_month,
day=day_match_value,
event_type=type(incarceration_event),
all_incarceration_events=all_incarceration_events)
if events_in_period and include_event_in_count(
incarceration_event, last_day_of_month(event_date),
events_in_period):
# Include this event in the person-based count
metrics.append((person_based_same_month_combo, 1))
return metrics
def map_recidivism_combinations(person: StatePerson,
release_events: Dict[int, List[ReleaseEvent]],
metric_inclusions: Dict[ReincarcerationRecidivismMetricType, bool]) \
-> List[Tuple[Dict[str, Any], Any]]:
"""Transforms ReleaseEvents and a StatePerson into metric combinations.
Takes in a StatePerson and all of her ReleaseEvents and returns an array
of "recidivism combinations". These are key-value pairs where the key
represents a specific metric and the value represents whether or not
recidivism occurred.
This translates a particular recidivism event into many different recidivism
metrics. Both count-based and rate-based metrics are generated. Each metric
represents one of many possible combinations of characteristics being
tracked for that event. For example, if an asian male is reincarcerated,
there is a metric that corresponds to asian people, one to males,
one to asian males, one to all people, and more depending on other
dimensions in the data.
If a release does not count towards recidivism, then the value is 0 for
the rate-based metrics in either methodology.
For both count and rate-based metrics, the value is 0 if the dimensions
of the metric do not fully match the attributes of the person and their type
of return to incarceration. For example, for a RecidivismReleaseEvent where
the return_type is 'REVOCATION', there will be metrics produced where the
return_type is 'NEW ADMISSION' and the value is 0.
Args:
person: the StatePerson
release_events: A dictionary mapping release cohorts to a list of
ReleaseEvents for the given StatePerson.
metric_inclusions: A dictionary where the keys are each ReincarcerationRecidivismMetricType, and the values
are boolean flags for whether or not to include that metric type in the calculations
Returns:
A list of key-value tuples representing specific metric combinations and
the recidivism value corresponding to that metric.
"""
metrics = []
all_reincarcerations = reincarcerations(release_events)
metric_period_end_date = last_day_of_month(date.today())
for release_cohort, events in release_events.items():
for event in events:
if metric_inclusions.get(ReincarcerationRecidivismMetricType.RATE):
characteristic_combo_rate = \
characteristics_dict(person, event, ReincarcerationRecidivismMetricType.RATE)
rate_metrics = map_recidivism_rate_combinations(
characteristic_combo_rate, release_cohort, event,
release_events, all_reincarcerations)
metrics.extend(rate_metrics)
if metric_inclusions.get(
ReincarcerationRecidivismMetricType.COUNT):
characteristic_combo_count = \
characteristics_dict(person, event, ReincarcerationRecidivismMetricType.COUNT)
count_metrics = map_recidivism_count_combinations(
characteristic_combo_count, event, all_reincarcerations,
metric_period_end_date)
metrics.extend(count_metrics)
return metrics
def find_time_buckets_for_supervision_period(
supervision_sentences: List[StateSupervisionSentence],
incarceration_sentences: List[StateIncarcerationSentence],
supervision_period: StateSupervisionPeriod,
incarceration_periods_by_admission_month: Dict[int, Dict[
int, List[StateIncarcerationPeriod]]],
months_fully_incarcerated: Set[Tuple[int, int]],
assessments: List[StateAssessment],
violation_responses: List[StateSupervisionViolationResponse],
supervision_period_to_agent_associations: Dict[int, Dict[Any, Any]]
) -> List[SupervisionTimeBucket]:
"""Finds months that this person was on supervision for the given StateSupervisionPeriod, where the person was not
incarcerated for the full month and did not have a revocation admission that month.
Args:
- supervision_period: The supervision period the person was on
- incarceration_periods_by_admission_month: A dictionary mapping years and months of admissions to prison to the
StateIncarcerationPeriods that started in that month.
- months_fully_incarcerated: A set of tuples in the format (year, month) for each month of which this person has
been incarcerated for the full month.
- ssvr_agent_associations: dictionary associating StateSupervisionViolationResponse ids to information about the
corresponding StateAgent on the response
Returns
- A set of unique SupervisionTimeBuckets for the person for the given StateSupervisionPeriod.
"""
supervision_month_buckets: List[SupervisionTimeBucket] = []
start_date = supervision_period.start_date
termination_date = supervision_period.termination_date
if termination_date is None:
termination_date = date.today()
if start_date is None:
return supervision_month_buckets
start_of_month = first_day_of_month(start_date)
while start_of_month <= termination_date:
if month_is_non_revocation_supervision_bucket(
start_of_month, termination_date, months_fully_incarcerated,
incarceration_periods_by_admission_month):
supervision_type = get_month_supervision_type(
start_of_month, supervision_sentences, incarceration_sentences,
supervision_period)
end_of_month = last_day_of_month(start_of_month)
assessment_score, assessment_level, assessment_type = find_most_recent_assessment(
end_of_month, assessments)
supervising_officer_external_id, supervising_district_external_id = \
_get_supervising_officer_and_district(supervision_period, supervision_period_to_agent_associations)
case_type = _identify_most_severe_case_type(supervision_period)
end_of_violation_window = end_of_month if end_of_month < termination_date else termination_date
violation_history = get_violation_and_response_history(
end_of_violation_window, violation_responses)
supervision_month_buckets.append(
NonRevocationReturnSupervisionTimeBucket(
state_code=supervision_period.state_code,
year=start_of_month.year,
month=start_of_month.month,
supervision_type=supervision_type,
case_type=case_type,
assessment_score=assessment_score,
assessment_level=assessment_level,
assessment_type=assessment_type,
most_severe_violation_type=violation_history.
most_severe_violation_type,
most_severe_violation_type_subtype=violation_history.
most_severe_violation_type_subtype,
response_count=violation_history.response_count,
supervising_officer_external_id=
supervising_officer_external_id,
supervising_district_external_id=
supervising_district_external_id,
supervision_level=supervision_period.supervision_level,
supervision_level_raw_text=supervision_period.
supervision_level_raw_text))
start_of_month = start_of_month + relativedelta(months=1)
return supervision_month_buckets
def map_supervision_combinations(
person: StatePerson,
supervision_time_buckets: List[SupervisionTimeBucket],
inclusions: Dict[str, bool],
calculation_month_limit: int) -> List[Tuple[Dict[str, Any], Any]]:
"""Transforms SupervisionTimeBuckets and a StatePerson into metric combinations.
Takes in a StatePerson and all of her SupervisionTimeBuckets and returns an array of "supervision combinations".
These are key-value pairs where the key represents a specific metric and the value represents whether or not
the person should be counted as a positive instance of that metric.
This translates a particular time on supervision into many different supervision population metrics. Each metric
represents one of many possible combinations of characteristics being tracked for that event. For example,
if a White male is on supervision, there is a metric that corresponds to White people, one to males, one to White
males, one to all people, and more depending on other dimensions in the data.
Args:
person: the StatePerson
supervision_time_buckets: A list of SupervisionTimeBuckets for the given StatePerson.
inclusions: A dictionary containing the following keys that correspond to characteristic dimensions:
- age_bucket
- ethnicity
- gender
- race
Where the values are boolean flags indicating whether to include the dimension in the calculations.
calculation_month_limit: The number of months (including this one) to limit the monthly calculation output to.
If set to -1, does not limit the calculations.
Returns:
A list of key-value tuples representing specific metric combinations and the value corresponding to that metric.
"""
metrics: List[Tuple[Dict[str, Any], Any]] = []
# We will calculate person-based metrics for each metric period in METRIC_PERIOD_MONTHS ending with the current
# month
metric_period_end_date = last_day_of_month(date.today())
calculation_month_lower_bound = get_calculation_month_lower_bound_date(
metric_period_end_date, calculation_month_limit)
supervision_time_buckets.sort(key=attrgetter('year', 'month'))
periods_and_buckets = _classify_buckets_by_relevant_metric_periods(
supervision_time_buckets, metric_period_end_date)
for supervision_time_bucket in supervision_time_buckets:
if isinstance(supervision_time_bucket,
ProjectedSupervisionCompletionBucket):
if inclusions.get(SupervisionMetricType.SUCCESS.value):
characteristic_combos_success = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.SUCCESS)
supervision_success_metrics = map_metric_combinations(
characteristic_combos_success, supervision_time_bucket,
metric_period_end_date, calculation_month_lower_bound,
supervision_time_buckets, periods_and_buckets,
SupervisionMetricType.SUCCESS)
metrics.extend(supervision_success_metrics)
if inclusions.get(SupervisionMetricType.SUCCESSFUL_SENTENCE_DAYS_SERVED.value) \
and supervision_time_bucket.successful_completion \
and not supervision_time_bucket.incarcerated_during_sentence:
# Only include successful sentences where the person was not incarcerated during the sentence in this
# metric
characteristic_combos_successful_sentence_length = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.SUCCESSFUL_SENTENCE_DAYS_SERVED)
successful_sentence_length_metrics = map_metric_combinations(
characteristic_combos_successful_sentence_length,
supervision_time_bucket, metric_period_end_date,
calculation_month_lower_bound, supervision_time_buckets,
periods_and_buckets,
SupervisionMetricType.SUCCESSFUL_SENTENCE_DAYS_SERVED)
metrics.extend(successful_sentence_length_metrics)
elif isinstance(supervision_time_bucket, SupervisionTerminationBucket):
if inclusions.get(SupervisionMetricType.ASSESSMENT_CHANGE.value):
characteristic_combos_assessment = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.ASSESSMENT_CHANGE)
assessment_change_metrics = map_metric_combinations(
characteristic_combos_assessment, supervision_time_bucket,
metric_period_end_date, calculation_month_lower_bound,
supervision_time_buckets, periods_and_buckets,
SupervisionMetricType.ASSESSMENT_CHANGE)
metrics.extend(assessment_change_metrics)
else:
if inclusions.get(SupervisionMetricType.POPULATION.value):
characteristic_combos_population = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.POPULATION)
population_metrics = map_metric_combinations(
characteristic_combos_population, supervision_time_bucket,
metric_period_end_date, calculation_month_lower_bound,
supervision_time_buckets, periods_and_buckets,
SupervisionMetricType.POPULATION)
metrics.extend(population_metrics)
if inclusions.get(SupervisionMetricType.REVOCATION.value):
characteristic_combos_revocation = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.REVOCATION)
if isinstance(supervision_time_bucket,
RevocationReturnSupervisionTimeBucket):
revocation_metrics = map_metric_combinations(
characteristic_combos_revocation,
supervision_time_bucket, metric_period_end_date,
calculation_month_lower_bound,
supervision_time_buckets, periods_and_buckets,
SupervisionMetricType.REVOCATION)
metrics.extend(revocation_metrics)
if inclusions.get(SupervisionMetricType.REVOCATION_ANALYSIS.value) and \
isinstance(supervision_time_bucket, RevocationReturnSupervisionTimeBucket):
characteristic_combos_revocation_analysis = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.REVOCATION_ANALYSIS)
revocation_analysis_metrics = map_metric_combinations(
characteristic_combos_revocation_analysis,
supervision_time_bucket, metric_period_end_date,
calculation_month_lower_bound, supervision_time_buckets,
periods_and_buckets,
SupervisionMetricType.REVOCATION_ANALYSIS)
metrics.extend(revocation_analysis_metrics)
if (inclusions.get(SupervisionMetricType.
REVOCATION_VIOLATION_TYPE_ANALYSIS.value)
and isinstance(supervision_time_bucket,
RevocationReturnSupervisionTimeBucket) and
supervision_time_bucket.violation_type_frequency_counter):
characteristic_combos_revocation_violation_type_analysis = characteristic_combinations(
person, supervision_time_bucket, inclusions,
SupervisionMetricType.REVOCATION_VIOLATION_TYPE_ANALYSIS)
revocation_violation_type_analysis_metrics = get_revocation_violation_type_analysis_metrics(
supervision_time_bucket,
characteristic_combos_revocation_violation_type_analysis,
metric_period_end_date, calculation_month_lower_bound,
supervision_time_buckets, periods_and_buckets)
metrics.extend(revocation_violation_type_analysis_metrics)
return metrics
def testClassifyIncarcerationEvents(self):
"""Tests the ClassifyIncarcerationEvents DoFn."""
fake_person_id = 12345
fake_person = StatePerson.new_with_defaults(
person_id=fake_person_id,
gender=Gender.MALE,
birthdate=date(1970, 1, 1),
residency_status=ResidencyStatus.PERMANENT)
incarceration_period = StateIncarcerationPeriod.new_with_defaults(
incarceration_period_id=1111,
incarceration_type=StateIncarcerationType.STATE_PRISON,
status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY,
state_code='TX',
facility='PRISON XX',
admission_date=date(2010, 11, 20),
admission_reason=StateIncarcerationPeriodAdmissionReason.
PROBATION_REVOCATION,
release_date=date(2010, 12, 4),
release_reason=StateIncarcerationPeriodReleaseReason.
SENTENCE_SERVED)
incarceration_sentence = StateIncarcerationSentence.new_with_defaults(
incarceration_sentence_id=123,
incarceration_periods=[incarceration_period],
charges=[
StateCharge.new_with_defaults(ncic_code='5699',
statute='CIVIL RIGHTS',
offense_date=date(2009, 1, 9))
])
sentence_group = StateSentenceGroup.new_with_defaults(
sentence_group_id=123,
incarceration_sentences=[incarceration_sentence])
incarceration_sentence.sentence_group = sentence_group
incarceration_period.incarceration_sentences = [incarceration_sentence]
person_entities = {
'person': [fake_person],
'sentence_groups': [sentence_group]
}
fake_person_id_to_county_query_result = [{
'person_id':
fake_person_id,
'county_of_residence':
_COUNTY_OF_RESIDENCE
}]
incarceration_events = [
IncarcerationStayEvent(
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.
admission_reason_raw_text,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PROBATION,
state_code=incarceration_period.state_code,
event_date=last_day_of_month(
incarceration_period.admission_date),
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
most_serious_offense_statute='CIVIL RIGHTS'),
IncarcerationAdmissionEvent(
state_code=incarceration_period.state_code,
event_date=incarceration_period.admission_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
admission_reason=incarceration_period.admission_reason,
admission_reason_raw_text=incarceration_period.
admission_reason_raw_text,
supervision_type_at_admission=
StateSupervisionPeriodSupervisionType.PROBATION,
),
IncarcerationReleaseEvent(
state_code=incarceration_period.state_code,
event_date=incarceration_period.release_date,
facility=incarceration_period.facility,
county_of_residence=_COUNTY_OF_RESIDENCE,
release_reason=incarceration_period.release_reason)
]
correct_output = [(fake_person, incarceration_events)]
test_pipeline = TestPipeline()
person_id_to_county_kv = (
test_pipeline
| "Read person id to county associations from BigQuery" >>
beam.Create(fake_person_id_to_county_query_result)
|
"Convert to KV" >> beam.ParDo(ConvertDictToKVTuple(), 'person_id'))
output = (test_pipeline
| beam.Create([(fake_person_id, person_entities)])
| 'Identify Incarceration Events' >> beam.ParDo(
pipeline.ClassifyIncarcerationEvents(),
AsDict(person_id_to_county_kv)))
assert_that(output, equal_to(correct_output))
test_pipeline.run()
def combination_incarceration_monthly_metrics(
combo: Dict[str, Any],
incarceration_event: IncarcerationEvent,
all_incarceration_events: List[IncarcerationEvent]) \
-> List[Tuple[Dict[str, Any], int]]:
"""Returns all unique incarceration metrics for the given event and combination.
First, includes an event-based count for the month the event occurred with a metric period of 1 month. Then, if
this event should be included in the person-based count for the month when the event occurred, adds those person-
based metrics.
Args:
combo: A characteristic combination to convert into metrics
incarceration_event: The IncarcerationEvent from which the combination was derived
all_incarceration_events: All of this person's IncarcerationEvents
Returns:
A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing
a positive contribution to that count metric.
"""
metrics = []
event_date = incarceration_event.event_date
event_year = event_date.year
event_month = event_date.month
# Add event-based combos for the 1-month period the month of the event
event_based_same_month_combos = augmented_combo_list(
combo, incarceration_event.state_code,
event_year, event_month,
MetricMethodologyType.EVENT, 1)
for event_combo in event_based_same_month_combos:
metrics.append((event_combo, 1))
# Create the person-based combos for the 1-month period of the month of the event
person_based_same_month_combos = augmented_combo_list(
combo, incarceration_event.state_code,
event_year, event_month,
MetricMethodologyType.PERSON, 1
)
events_in_month: List[IncarcerationEvent] = []
if isinstance(incarceration_event, IncarcerationAdmissionEvent):
# All admission events that happened the same month as this one
events_in_month = [
event for event in all_incarceration_events
if isinstance(event, IncarcerationAdmissionEvent) and
event.event_date.year == event_date.year and
event.event_date.month == event_date.month
]
elif isinstance(incarceration_event, IncarcerationStayEvent):
# All stay events that happened the same month as this one
events_in_month = [
event for event in all_incarceration_events
if isinstance(event, IncarcerationStayEvent) and
event.event_date.year == event_date.year and
event.event_date.month == event_date.month
]
elif isinstance(incarceration_event, IncarcerationReleaseEvent):
# All release events that happened the same month as this one
events_in_month = [
event for event in all_incarceration_events
if isinstance(event, IncarcerationReleaseEvent) and
event.event_date.year == event_date.year and
event.event_date.month == event_date.month
]
if events_in_month and include_event_in_count(
incarceration_event,
last_day_of_month(event_date),
events_in_month):
# Include this event in the person-based count
for person_combo in person_based_same_month_combos:
metrics.append((person_combo, 1))
return metrics
def find_revocation_return_buckets(
supervision_sentences: List[StateSupervisionSentence],
incarceration_sentences: List[StateIncarcerationSentence],
supervision_periods: List[StateSupervisionPeriod],
incarceration_periods: List[StateIncarcerationPeriod],
supervision_time_buckets: List[SupervisionTimeBucket],
assessments: List[StateAssessment],
violation_responses: List[StateSupervisionViolationResponse],
ssvr_agent_associations: Dict[int, Dict[Any, Any]],
supervision_period_to_agent_associations: Dict[int, Dict[Any, Any]]) -> \
List[SupervisionTimeBucket]:
"""Looks at all incarceration periods to see if they were revocation returns. For each revocation admission, adds
one RevocationReturnSupervisionTimeBuckets for each overlapping supervision period. If there are no overlapping
supervision periods, looks for a recently terminated period. If there are no overlapping or recently terminated
supervision periods, adds a RevocationReturnSupervisionTimeBuckets with as much information about the time on
supervision as possible.
"""
revocation_return_buckets: List[SupervisionTimeBucket] = []
for incarceration_period in incarceration_periods:
if not incarceration_period.admission_date:
raise ValueError(
f"Admission date for null for {incarceration_period}")
if not is_revocation_admission(incarceration_period.admission_reason):
continue
admission_date = incarceration_period.admission_date
admission_year = admission_date.year
admission_month = admission_date.month
end_of_month = last_day_of_month(admission_date)
assessment_score, assessment_level, assessment_type = find_most_recent_assessment(
end_of_month, assessments)
relevant_pre_incarceration_supervision_periods = \
_get_relevant_supervision_periods_before_admission_date(admission_date, supervision_periods)
if relevant_pre_incarceration_supervision_periods:
# Add a RevocationReturnSupervisionTimeBucket for each overlapping supervision period
for supervision_period in relevant_pre_incarceration_supervision_periods:
revocation_details = _get_revocation_details(
incarceration_period, supervision_period,
ssvr_agent_associations,
supervision_period_to_agent_associations)
supervision_type_at_admission = get_pre_incarceration_supervision_type(
incarceration_sentences, supervision_sentences,
incarceration_period, [supervision_period])
case_type = _identify_most_severe_case_type(supervision_period)
supervision_level = supervision_period.supervision_level
supervision_level_raw_text = supervision_period.supervision_level_raw_text
# Get details about the violation and response history leading up to the revocation
violation_history = get_violation_and_response_history(
admission_date, violation_responses)
revocation_month_bucket = RevocationReturnSupervisionTimeBucket(
state_code=incarceration_period.state_code,
year=admission_year,
month=admission_month,
supervision_type=supervision_type_at_admission,
case_type=case_type,
assessment_score=assessment_score,
assessment_level=assessment_level,
assessment_type=assessment_type,
revocation_type=revocation_details.revocation_type,
source_violation_type=revocation_details.
source_violation_type,
most_severe_violation_type=violation_history.
most_severe_violation_type,
most_severe_violation_type_subtype=violation_history.
most_severe_violation_type_subtype,
most_severe_response_decision=violation_history.
most_severe_response_decision,
response_count=violation_history.response_count,
violation_history_description=violation_history.
violation_history_description,
violation_type_frequency_counter=violation_history.
violation_type_frequency_counter,
supervising_officer_external_id=revocation_details.
supervising_officer_external_id,
supervising_district_external_id=revocation_details.
supervising_district_external_id,
supervision_level=supervision_level,
supervision_level_raw_text=supervision_level_raw_text)
revocation_return_buckets.append(revocation_month_bucket)
else:
# There are no overlapping or proximal supervision periods. Add one
# RevocationReturnSupervisionTimeBucket with as many details as possible about this revocation
revocation_details = _get_revocation_details(
incarceration_period, None, ssvr_agent_associations, None)
supervision_type_at_admission = get_pre_incarceration_supervision_type(
incarceration_sentences, supervision_sentences,
incarceration_period, [])
# TODO(2853): Don't default to GENERAL once we figure out how to handle unset fields
case_type = StateSupervisionCaseType.GENERAL
end_of_month = last_day_of_month(admission_date)
assessment_score, assessment_level, assessment_type = find_most_recent_assessment(
end_of_month, assessments)
# Get details about the violation and response history leading up to the revocation
violation_history = get_violation_and_response_history(
admission_date, violation_responses)
if supervision_type_at_admission is not None:
revocation_month_bucket = RevocationReturnSupervisionTimeBucket(
state_code=incarceration_period.state_code,
year=admission_year,
month=admission_month,
supervision_type=supervision_type_at_admission,
case_type=case_type,
assessment_score=assessment_score,
assessment_level=assessment_level,
assessment_type=assessment_type,
revocation_type=revocation_details.revocation_type,
source_violation_type=revocation_details.
source_violation_type,
most_severe_violation_type=violation_history.
most_severe_violation_type,
most_severe_violation_type_subtype=violation_history.
most_severe_violation_type_subtype,
most_severe_response_decision=violation_history.
most_severe_response_decision,
response_count=violation_history.response_count,
violation_history_description=violation_history.
violation_history_description,
violation_type_frequency_counter=violation_history.
violation_type_frequency_counter,
supervising_officer_external_id=revocation_details.
supervising_officer_external_id,
supervising_district_external_id=revocation_details.
supervising_district_external_id)
revocation_return_buckets.append(revocation_month_bucket)
if revocation_return_buckets:
supervision_time_buckets.extend(revocation_return_buckets)
return supervision_time_buckets
