def month_is_non_revocation_supervision_bucket( start_of_month: date, termination_date: date, months_fully_incarcerated: Set[Tuple[int, int]], incarceration_periods_by_admission_month: Dict[int, Dict[ int, List[StateIncarcerationPeriod]]]): """Determines whether the given month was a month on supervision without a revocation and without being incarcerated for the full time spent on supervision that month.""" was_incarcerated_all_month = ( start_of_month.year, start_of_month.month) in months_fully_incarcerated if was_incarcerated_all_month: return False if has_revocation_admission_in_month( start_of_month, incarceration_periods_by_admission_month): return False if last_day_of_month(termination_date) == last_day_of_month( start_of_month): # If the supervision period ended this month, make sure there wasn't an incarceration period that # fully overlapped with the days of supervision in this month incarceration_overlapping_supervision_this_month = [ ip for months in incarceration_periods_by_admission_month.values() for ips_in_month in months.values() for ip in ips_in_month if ip.admission_date and ip.admission_date <= start_of_month and ( ip.release_date is None or termination_date <= ip.release_date) ] if incarceration_overlapping_supervision_this_month: return False return True
def find_end_of_month_state_prison_stays( incarceration_sentences: List[StateIncarcerationSentence], supervision_sentences: List[StateSupervisionSentence], incarceration_period: StateIncarcerationPeriod, county_of_residence: Optional[str]) -> List[IncarcerationStayEvent]: """Finds months for which this person was incarcerated in a state prison on the last day of the month. """ incarceration_stay_events: List[IncarcerationStayEvent] = [] if incarceration_period.incarceration_type != StateIncarcerationType.STATE_PRISON: return incarceration_stay_events admission_date = incarceration_period.admission_date release_date = incarceration_period.release_date if release_date is None: release_date = date.today() if admission_date is None: return incarceration_stay_events supervision_type_at_admission = get_pre_incarceration_supervision_type( incarceration_sentences, supervision_sentences, incarceration_period) sentence_group = _get_sentence_group_for_incarceration_period(incarceration_period) end_of_month = last_day_of_month(admission_date) while end_of_month < release_date: most_serious_charge = find_most_serious_prior_charge_in_sentence_group(sentence_group, end_of_month) most_serious_offense_ncic_code = most_serious_charge.ncic_code if most_serious_charge else None most_serious_offense_statute = most_serious_charge.statute if most_serious_charge else None incarceration_stay_events.append( IncarcerationStayEvent( state_code=incarceration_period.state_code, event_date=end_of_month, facility=incarceration_period.facility, county_of_residence=county_of_residence, most_serious_offense_ncic_code=most_serious_offense_ncic_code, most_serious_offense_statute=most_serious_offense_statute, admission_reason=incarceration_period.admission_reason, admission_reason_raw_text=incarceration_period.admission_reason_raw_text, supervision_type_at_admission=supervision_type_at_admission, ) ) end_of_month = last_day_of_month(end_of_month + relativedelta(days=1)) return incarceration_stay_events
def _identify_months_fully_incarcerated( incarceration_periods: List[StateIncarcerationPeriod]) -> \ Set[Tuple[int, int]]: """For each StateIncarcerationPeriod, identifies months where the person was incarcerated for every day during that month. Returns a set of months in the format (year, month) for which the person spent the entire month in a prison.""" months_incarcerated: Set[Tuple[int, int]] = set() for incarceration_period in incarceration_periods: admission_date = incarceration_period.admission_date release_date = incarceration_period.release_date if admission_date is None: return months_incarcerated if release_date is None: release_date = last_day_of_month(date.today()) if admission_date.day == 1: month_date = admission_date else: month_date = first_day_of_month(admission_date) + relativedelta( months=1) while month_date + relativedelta( months=1) <= release_date + relativedelta(days=1): months_incarcerated.add((month_date.year, month_date.month)) month_date = month_date + relativedelta(months=1) return months_incarcerated
def us_mo_get_month_supervision_type( any_date_in_month: datetime.date, supervision_sentences: List[StateSupervisionSentence], incarceration_sentences: List[StateIncarcerationSentence], supervision_period: StateSupervisionPeriod ) -> StateSupervisionPeriodSupervisionType: """Calculates the supervision period supervision type that should be attributed to a US_MO supervision period on a given month. The date used to calculate the supervision period supervision type is either the last day of the month, or the last day of supervision, whichever comes first. """ end_of_month = last_day_of_month(any_date_in_month) if supervision_period.termination_date is None: supervision_type_determination_date = end_of_month else: supervision_type_determination_date = min( end_of_month, supervision_period.termination_date - datetime.timedelta(days=1)) supervision_type = \ us_mo_get_supervision_period_supervision_type_on_date(supervision_type_determination_date, supervision_sentences, incarceration_sentences) if not supervision_type: return StateSupervisionPeriodSupervisionType.INTERNAL_UNKNOWN return supervision_type
def us_mo_get_month_supervision_type( any_date_in_month: datetime.date, supervision_sentences: List[StateSupervisionSentence], incarceration_sentences: List[StateIncarcerationSentence], supervision_period: StateSupervisionPeriod ) -> StateSupervisionPeriodSupervisionType: """Calculates the supervision period supervision type that should be attributed to a US_MO supervision period on a given month. The date used to calculate the supervision period supervision type is either the last day of the month, or the last day of supervision, whichever comes first. """ start_of_month = first_day_of_month(any_date_in_month) end_of_month = last_day_of_month(any_date_in_month) first_of_next_month = end_of_month + datetime.timedelta(days=1) if supervision_period.termination_date is None: upper_bound_exclusive_date = first_of_next_month else: upper_bound_exclusive_date = min(first_of_next_month, supervision_period.termination_date) lower_bound_inclusive = max(start_of_month, supervision_period.start_date) supervision_type = \ us_mo_get_most_recent_supervision_period_supervision_type_before_upper_bound_day( upper_bound_exclusive_date=upper_bound_exclusive_date, lower_bound_inclusive_date=lower_bound_inclusive, supervision_sentences=supervision_sentences, incarceration_sentences=incarceration_sentences) if not supervision_type: return StateSupervisionPeriodSupervisionType.INTERNAL_UNKNOWN return supervision_type
def combination_referral_monthly_metrics( combo: Dict[str, Any], program_event: ProgramReferralEvent, all_referral_events: List[ProgramReferralEvent]) \ -> List[Tuple[Dict[str, Any], int]]: """Returns all unique referral metrics for the given event and combination. First, includes an event-based count for the month the event occurred with a metric period of 1 month. Then, if this event should be included in the person-based count for the month when the event occurred, adds those person- based metrics. Args: combo: A characteristic combination to convert into metrics program_event: The program event from which the combination was derived all_referral_events: All of this person's ProgramReferralEvents Returns: A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing a positive contribution to that count metric. """ metrics = [] event_date = program_event.event_date event_year = event_date.year event_month = event_date.month # Add event-based combo for the 1-month period the month of the event event_based_same_month_combo = augmented_combo_for_calculations( combo, program_event.state_code, event_year, event_month, MetricMethodologyType.EVENT, 1) metrics.append((event_based_same_month_combo, 1)) # Create the person-based combo for the 1-month period of the month of the event person_based_same_month_combo = augmented_combo_for_calculations( combo, program_event.state_code, event_year, event_month, MetricMethodologyType.PERSON, 1 ) # Get all other referral events that happened the same month as this one all_referral_events_in_event_month = [ event for event in all_referral_events if event.event_date.year == event_date.year and event.event_date.month == event_date.month ] if include_referral_in_count( combo, program_event, last_day_of_month(event_date), all_referral_events_in_event_month): # Include this event in the person-based count metrics.append((person_based_same_month_combo, 1)) return metrics
def map_recidivism_count_combinations( characteristic_combos: List[Dict[str, Any]], event: ReleaseEvent, all_reincarcerations: Dict[date, Dict[str, Any]], metric_period_end_date: date) -> \ List[Tuple[Dict[str, Any], Any]]: """Maps the given event and characteristic combinations to a variety of metrics that track count-based recidivism. If the event is a RecidivismReleaseEvent, then a count of reincarceration occurred. This produces metrics for both the year and the month in which the person was reincarcerated. Args: characteristic_combos: A list of dictionaries containing all unique combinations of characteristics. event: the recidivism event from which the combination was derived all_reincarcerations: dictionary where the keys are all dates of reincarceration for the person's ReleaseEvents, and the values are a dictionary containing return type and from supervision type information metric_period_end_date: The day the metric periods end Returns: A list of key-value tuples representing specific metric combinations and the recidivism value corresponding to that metric. """ metrics = [] if isinstance(event, RecidivismReleaseEvent): reincarceration_date = event.reincarceration_date relevant_periods = relevant_metric_periods( reincarceration_date, metric_period_end_date.year, metric_period_end_date.month) for combo in characteristic_combos: combo['metric_type'] = ReincarcerationRecidivismMetricType.COUNT # Bucket for the month of the incarceration combo['year'] = reincarceration_date.year combo['month'] = reincarceration_date.month combo['metric_period_months'] = 1 end_of_event_month = last_day_of_month(reincarceration_date) metrics.extend( combination_count_metrics(combo, event, all_reincarcerations, end_of_event_month)) # Bucket for each of the relevant metric period month lengths for relevant_period in relevant_periods: combo['year'] = metric_period_end_date.year combo['month'] = metric_period_end_date.month combo['metric_period_months'] = relevant_period metrics.extend( combination_count_metrics(combo, event, all_reincarcerations, metric_period_end_date)) return metrics
def map_recidivism_liberty_combinations( characteristic_combos: List[Dict[str, Any]], event: ReleaseEvent, all_reincarcerations: Dict[date, Dict[str, Any]]) -> \ List[Tuple[Dict[str, Any], Any]]: """Maps the given event and characteristic combinations to a variety of metrics that track metrics for time at liberty. """ metrics = [] if isinstance(event, RecidivismReleaseEvent): reincarceration_date = event.reincarceration_date year = reincarceration_date.year year_start_day = date(year, 1, 1) year_end_day = date(year, 12, 31) month_start_day = first_day_of_month(reincarceration_date) month_end_day = last_day_of_month(reincarceration_date) for combo in characteristic_combos: combo['metric_type'] = ReincarcerationRecidivismMetricType.LIBERTY # Year bucket combo['start_date'] = year_start_day combo['end_date'] = year_end_day metrics.extend( combination_liberty_metrics(combo, event, all_reincarcerations)) # Month bucket combo['start_date'] = month_start_day combo['end_date'] = month_end_day metrics.extend( combination_liberty_metrics(combo, event, all_reincarcerations)) return metrics
def get_month_supervision_type( any_date_in_month: datetime.date, supervision_sentences: List[StateSupervisionSentence], incarceration_sentences: List[StateIncarcerationSentence], supervision_period: StateSupervisionPeriod ) -> StateSupervisionPeriodSupervisionType: """Supervision type can change over time even if the period does not change. This function calculates the supervision type that a given supervision period represents during the month that |any_date_in_month| falls in. We do this by looking at all sentences attached to this supervision period, then determining which ones overlap with any day in the month, and using the sentence supervision types to determine the period supervision type at this point in time. Args: any_date_in_month: (date) Any day in the month to consider supervision_period: (StateSupervisionPeriod) The supervision period we want to associate a supervision type with supervision_sentences: (List[StateSupervisionSentence]) All supervision sentences for a given person. """ if not supervision_period.supervision_period_id: raise ValueError('All objects should have database ids.') if is_placeholder(supervision_period): raise ValueError('Do not expect placeholder periods!') start_of_month = first_day_of_month(any_date_in_month) end_of_month = last_day_of_month(any_date_in_month) # Find sentences that are attached to the period and overlap with the month incarceration_sentences = _get_valid_attached_sentences( incarceration_sentences, supervision_period) incarceration_sentences = _get_sentences_overlapping_with_dates( start_of_month, end_of_month, incarceration_sentences) supervision_sentences = _get_valid_attached_sentences( supervision_sentences, supervision_period) supervision_sentences = _get_sentences_overlapping_with_dates( start_of_month, end_of_month, supervision_sentences) return _get_supervision_type_from_sentences(incarceration_sentences, supervision_sentences)
def map_incarceration_combinations(person: StatePerson, incarceration_events: List[IncarcerationEvent], inclusions: Dict[str, bool], calculation_month_limit: int) -> List[Tuple[Dict[str, Any], Any]]: """Transforms IncarcerationEvents and a StatePerson into metric combinations. Takes in a StatePerson and all of their IncarcerationEvent and returns an array of "incarceration combinations". These are key-value pairs where the key represents a specific metric and the value represents whether or not the person should be counted as a positive instance of that metric. This translates a particular incarceration event, e.g. admission or release, into many different incarceration metrics. Each metric represents one of many possible combinations of characteristics being tracked for that event. For example, if a White male is admitted to prison, there is a metric that corresponds to White people, one to males, one to White males, one to all people, and more depending on other dimensions in the data. Args: person: the StatePerson incarceration_events: A list of IncarcerationEvents for the given StatePerson. inclusions: A dictionary containing the following keys that correspond to characteristic dimensions: - age_bucket - ethnicity - gender - race Where the values are boolean flags indicating whether to include the dimension in the calculations. calculation_month_limit: The number of months (including this one) to limit the monthly calculation output to. If set to -1, does not limit the calculations. Returns: A list of key-value tuples representing specific metric combinations and the value corresponding to that metric. """ metrics: List[Tuple[Dict[str, Any], Any]] = [] periods_and_events: Dict[int, List[IncarcerationEvent]] = defaultdict() # We will calculate person-based metrics for each metric period in METRIC_PERIOD_MONTHS ending with the current # month metric_period_end_date = last_day_of_month(date.today()) calculation_month_lower_bound = get_calculation_month_lower_bound_date( metric_period_end_date, calculation_month_limit) # Organize the events by the relevant metric periods for incarceration_event in incarceration_events: relevant_periods = relevant_metric_periods( incarceration_event.event_date, metric_period_end_date.year, metric_period_end_date.month) if relevant_periods: for period in relevant_periods: period_events = periods_and_events.get(period) if period_events: period_events.append(incarceration_event) else: periods_and_events[period] = [incarceration_event] for incarceration_event in incarceration_events: metric_type = METRIC_TYPES.get(type(incarceration_event)) if not metric_type: raise ValueError( 'No metric type mapped to incarceration event of type {}'.format(type(incarceration_event))) characteristic_combos = characteristic_combinations(person, incarceration_event, inclusions, metric_type) metrics.extend(map_metric_combinations( characteristic_combos, incarceration_event, metric_period_end_date, calculation_month_lower_bound, incarceration_events, periods_and_events, metric_type )) return metrics
def combination_program_monthly_metrics( combo: Dict[str, Any], program_event: ProgramEvent, metric_type: ProgramMetricType, all_program_events: List[ProgramEvent], is_daily_metric: bool) -> List[Tuple[Dict[str, Any], int]]: """Returns all unique referral metrics for the given event and combination. First, includes an event-based count for the month the event occurred with a metric period of 1 month. Then, if this event should be included in the person-based count for the month when the event occurred, adds those person- based metrics. Args: combo: A characteristic combination to convert into metrics program_event: The program event from which the combination was derived metric_type: The type of metric being tracked by this combo all_program_events: All of this person's ProgramEvents is_daily_metric: If True, limits person-based counts to the date of the event. If False, limits person-based counts to the month of the event. Returns: A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing a positive contribution to that count metric. """ metrics = [] event_date = program_event.event_date event_year = event_date.year event_month = event_date.month base_metric_period = 0 if is_daily_metric else 1 # Add event-based combo for the base metric period the month of the event event_based_same_month_combo = augmented_combo_for_calculations( combo, program_event.state_code, event_year, event_month, MetricMethodologyType.EVENT, base_metric_period) metrics.append((event_based_same_month_combo, 1)) # Create the person-based combo for the base metric period of the month of the event person_based_same_month_combo = augmented_combo_for_calculations( combo, program_event.state_code, event_year, event_month, MetricMethodologyType.PERSON, base_metric_period) events_in_period: List[ProgramEvent] = [] if metric_type == ProgramMetricType.PARTICIPATION: # Get all other participation events that happened on the same day as this one events_in_period = [ event for event in all_program_events if (isinstance(event, ProgramParticipationEvent)) and event.event_date == program_event.event_date ] elif metric_type == ProgramMetricType.REFERRAL: # Get all other referral events that happened in the same month as this one events_in_period = [ event for event in all_program_events if isinstance(event, ProgramReferralEvent) and event.event_date. year == event_year and event.event_date.month == event_month ] if include_event_in_count(combo, program_event, last_day_of_month(event_date), events_in_period): # Include this event in the person-based count metrics.append((person_based_same_month_combo, 1)) return metrics
def combination_incarceration_metrics( combo: Dict[str, Any], incarceration_event: IncarcerationEvent, all_incarceration_events: List[IncarcerationEvent], is_daily_metric: bool) \ -> List[Tuple[Dict[str, Any], int]]: """Returns all unique incarceration metrics for the given event and combination. First, includes an event-based count for the event. Then, if this is a daily metric, includes a count of the event if it should be included in the person-based count for the day when the event occurred. If this is not a daily metric, includes a count of the event if it should be included in the person-based count for the month of the event. Args: combo: A characteristic combination to convert into metrics incarceration_event: The IncarcerationEvent from which the combination was derived all_incarceration_events: All of this person's IncarcerationEvents is_daily_metric: If True, limits person-based counts to the date of the event. If False, limits person-based counts to the month of the event. Returns: A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing a positive contribution to that count metric. """ metrics = [] event_date = incarceration_event.event_date event_year = event_date.year event_month = event_date.month metric_period_months = 0 if is_daily_metric else 1 # Add event-based combo for the 1-month period the month of the event event_based_same_month_combo = augmented_combo_for_calculations( combo, incarceration_event.state_code, event_year, event_month, MetricMethodologyType.EVENT, metric_period_months=metric_period_months) metrics.append((event_based_same_month_combo, 1)) # Create the person-based combo for the 1-month period of the month of the event person_based_same_month_combo = augmented_combo_for_calculations( combo, incarceration_event.state_code, event_year, event_month, MetricMethodologyType.PERSON, metric_period_months=metric_period_months) day_match_value = event_date.day if is_daily_metric else None # Get the events of the same type that happened in the same month events_in_period = matching_events_for_person_based_count( year=event_year, month=event_month, day=day_match_value, event_type=type(incarceration_event), all_incarceration_events=all_incarceration_events) if events_in_period and include_event_in_count( incarceration_event, last_day_of_month(event_date), events_in_period): # Include this event in the person-based count metrics.append((person_based_same_month_combo, 1)) return metrics
def map_recidivism_combinations(person: StatePerson, release_events: Dict[int, List[ReleaseEvent]], metric_inclusions: Dict[ReincarcerationRecidivismMetricType, bool]) \ -> List[Tuple[Dict[str, Any], Any]]: """Transforms ReleaseEvents and a StatePerson into metric combinations. Takes in a StatePerson and all of her ReleaseEvents and returns an array of "recidivism combinations". These are key-value pairs where the key represents a specific metric and the value represents whether or not recidivism occurred. This translates a particular recidivism event into many different recidivism metrics. Both count-based and rate-based metrics are generated. Each metric represents one of many possible combinations of characteristics being tracked for that event. For example, if an asian male is reincarcerated, there is a metric that corresponds to asian people, one to males, one to asian males, one to all people, and more depending on other dimensions in the data. If a release does not count towards recidivism, then the value is 0 for the rate-based metrics in either methodology. For both count and rate-based metrics, the value is 0 if the dimensions of the metric do not fully match the attributes of the person and their type of return to incarceration. For example, for a RecidivismReleaseEvent where the return_type is 'REVOCATION', there will be metrics produced where the return_type is 'NEW ADMISSION' and the value is 0. Args: person: the StatePerson release_events: A dictionary mapping release cohorts to a list of ReleaseEvents for the given StatePerson. metric_inclusions: A dictionary where the keys are each ReincarcerationRecidivismMetricType, and the values are boolean flags for whether or not to include that metric type in the calculations Returns: A list of key-value tuples representing specific metric combinations and the recidivism value corresponding to that metric. """ metrics = [] all_reincarcerations = reincarcerations(release_events) metric_period_end_date = last_day_of_month(date.today()) for release_cohort, events in release_events.items(): for event in events: if metric_inclusions.get(ReincarcerationRecidivismMetricType.RATE): characteristic_combo_rate = \ characteristics_dict(person, event, ReincarcerationRecidivismMetricType.RATE) rate_metrics = map_recidivism_rate_combinations( characteristic_combo_rate, release_cohort, event, release_events, all_reincarcerations) metrics.extend(rate_metrics) if metric_inclusions.get( ReincarcerationRecidivismMetricType.COUNT): characteristic_combo_count = \ characteristics_dict(person, event, ReincarcerationRecidivismMetricType.COUNT) count_metrics = map_recidivism_count_combinations( characteristic_combo_count, event, all_reincarcerations, metric_period_end_date) metrics.extend(count_metrics) return metrics
def find_time_buckets_for_supervision_period( supervision_sentences: List[StateSupervisionSentence], incarceration_sentences: List[StateIncarcerationSentence], supervision_period: StateSupervisionPeriod, incarceration_periods_by_admission_month: Dict[int, Dict[ int, List[StateIncarcerationPeriod]]], months_fully_incarcerated: Set[Tuple[int, int]], assessments: List[StateAssessment], violation_responses: List[StateSupervisionViolationResponse], supervision_period_to_agent_associations: Dict[int, Dict[Any, Any]] ) -> List[SupervisionTimeBucket]: """Finds months that this person was on supervision for the given StateSupervisionPeriod, where the person was not incarcerated for the full month and did not have a revocation admission that month. Args: - supervision_period: The supervision period the person was on - incarceration_periods_by_admission_month: A dictionary mapping years and months of admissions to prison to the StateIncarcerationPeriods that started in that month. - months_fully_incarcerated: A set of tuples in the format (year, month) for each month of which this person has been incarcerated for the full month. - ssvr_agent_associations: dictionary associating StateSupervisionViolationResponse ids to information about the corresponding StateAgent on the response Returns - A set of unique SupervisionTimeBuckets for the person for the given StateSupervisionPeriod. """ supervision_month_buckets: List[SupervisionTimeBucket] = [] start_date = supervision_period.start_date termination_date = supervision_period.termination_date if termination_date is None: termination_date = date.today() if start_date is None: return supervision_month_buckets start_of_month = first_day_of_month(start_date) while start_of_month <= termination_date: if month_is_non_revocation_supervision_bucket( start_of_month, termination_date, months_fully_incarcerated, incarceration_periods_by_admission_month): supervision_type = get_month_supervision_type( start_of_month, supervision_sentences, incarceration_sentences, supervision_period) end_of_month = last_day_of_month(start_of_month) assessment_score, assessment_level, assessment_type = find_most_recent_assessment( end_of_month, assessments) supervising_officer_external_id, supervising_district_external_id = \ _get_supervising_officer_and_district(supervision_period, supervision_period_to_agent_associations) case_type = _identify_most_severe_case_type(supervision_period) end_of_violation_window = end_of_month if end_of_month < termination_date else termination_date violation_history = get_violation_and_response_history( end_of_violation_window, violation_responses) supervision_month_buckets.append( NonRevocationReturnSupervisionTimeBucket( state_code=supervision_period.state_code, year=start_of_month.year, month=start_of_month.month, supervision_type=supervision_type, case_type=case_type, assessment_score=assessment_score, assessment_level=assessment_level, assessment_type=assessment_type, most_severe_violation_type=violation_history. most_severe_violation_type, most_severe_violation_type_subtype=violation_history. most_severe_violation_type_subtype, response_count=violation_history.response_count, supervising_officer_external_id= supervising_officer_external_id, supervising_district_external_id= supervising_district_external_id, supervision_level=supervision_period.supervision_level, supervision_level_raw_text=supervision_period. supervision_level_raw_text)) start_of_month = start_of_month + relativedelta(months=1) return supervision_month_buckets
def map_supervision_combinations( person: StatePerson, supervision_time_buckets: List[SupervisionTimeBucket], inclusions: Dict[str, bool], calculation_month_limit: int) -> List[Tuple[Dict[str, Any], Any]]: """Transforms SupervisionTimeBuckets and a StatePerson into metric combinations. Takes in a StatePerson and all of her SupervisionTimeBuckets and returns an array of "supervision combinations". These are key-value pairs where the key represents a specific metric and the value represents whether or not the person should be counted as a positive instance of that metric. This translates a particular time on supervision into many different supervision population metrics. Each metric represents one of many possible combinations of characteristics being tracked for that event. For example, if a White male is on supervision, there is a metric that corresponds to White people, one to males, one to White males, one to all people, and more depending on other dimensions in the data. Args: person: the StatePerson supervision_time_buckets: A list of SupervisionTimeBuckets for the given StatePerson. inclusions: A dictionary containing the following keys that correspond to characteristic dimensions: - age_bucket - ethnicity - gender - race Where the values are boolean flags indicating whether to include the dimension in the calculations. calculation_month_limit: The number of months (including this one) to limit the monthly calculation output to. If set to -1, does not limit the calculations. Returns: A list of key-value tuples representing specific metric combinations and the value corresponding to that metric. """ metrics: List[Tuple[Dict[str, Any], Any]] = [] # We will calculate person-based metrics for each metric period in METRIC_PERIOD_MONTHS ending with the current # month metric_period_end_date = last_day_of_month(date.today()) calculation_month_lower_bound = get_calculation_month_lower_bound_date( metric_period_end_date, calculation_month_limit) supervision_time_buckets.sort(key=attrgetter('year', 'month')) periods_and_buckets = _classify_buckets_by_relevant_metric_periods( supervision_time_buckets, metric_period_end_date) for supervision_time_bucket in supervision_time_buckets: if isinstance(supervision_time_bucket, ProjectedSupervisionCompletionBucket): if inclusions.get(SupervisionMetricType.SUCCESS.value): characteristic_combos_success = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.SUCCESS) supervision_success_metrics = map_metric_combinations( characteristic_combos_success, supervision_time_bucket, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets, SupervisionMetricType.SUCCESS) metrics.extend(supervision_success_metrics) if inclusions.get(SupervisionMetricType.SUCCESSFUL_SENTENCE_DAYS_SERVED.value) \ and supervision_time_bucket.successful_completion \ and not supervision_time_bucket.incarcerated_during_sentence: # Only include successful sentences where the person was not incarcerated during the sentence in this # metric characteristic_combos_successful_sentence_length = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.SUCCESSFUL_SENTENCE_DAYS_SERVED) successful_sentence_length_metrics = map_metric_combinations( characteristic_combos_successful_sentence_length, supervision_time_bucket, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets, SupervisionMetricType.SUCCESSFUL_SENTENCE_DAYS_SERVED) metrics.extend(successful_sentence_length_metrics) elif isinstance(supervision_time_bucket, SupervisionTerminationBucket): if inclusions.get(SupervisionMetricType.ASSESSMENT_CHANGE.value): characteristic_combos_assessment = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.ASSESSMENT_CHANGE) assessment_change_metrics = map_metric_combinations( characteristic_combos_assessment, supervision_time_bucket, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets, SupervisionMetricType.ASSESSMENT_CHANGE) metrics.extend(assessment_change_metrics) else: if inclusions.get(SupervisionMetricType.POPULATION.value): characteristic_combos_population = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.POPULATION) population_metrics = map_metric_combinations( characteristic_combos_population, supervision_time_bucket, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets, SupervisionMetricType.POPULATION) metrics.extend(population_metrics) if inclusions.get(SupervisionMetricType.REVOCATION.value): characteristic_combos_revocation = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.REVOCATION) if isinstance(supervision_time_bucket, RevocationReturnSupervisionTimeBucket): revocation_metrics = map_metric_combinations( characteristic_combos_revocation, supervision_time_bucket, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets, SupervisionMetricType.REVOCATION) metrics.extend(revocation_metrics) if inclusions.get(SupervisionMetricType.REVOCATION_ANALYSIS.value) and \ isinstance(supervision_time_bucket, RevocationReturnSupervisionTimeBucket): characteristic_combos_revocation_analysis = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.REVOCATION_ANALYSIS) revocation_analysis_metrics = map_metric_combinations( characteristic_combos_revocation_analysis, supervision_time_bucket, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets, SupervisionMetricType.REVOCATION_ANALYSIS) metrics.extend(revocation_analysis_metrics) if (inclusions.get(SupervisionMetricType. REVOCATION_VIOLATION_TYPE_ANALYSIS.value) and isinstance(supervision_time_bucket, RevocationReturnSupervisionTimeBucket) and supervision_time_bucket.violation_type_frequency_counter): characteristic_combos_revocation_violation_type_analysis = characteristic_combinations( person, supervision_time_bucket, inclusions, SupervisionMetricType.REVOCATION_VIOLATION_TYPE_ANALYSIS) revocation_violation_type_analysis_metrics = get_revocation_violation_type_analysis_metrics( supervision_time_bucket, characteristic_combos_revocation_violation_type_analysis, metric_period_end_date, calculation_month_lower_bound, supervision_time_buckets, periods_and_buckets) metrics.extend(revocation_violation_type_analysis_metrics) return metrics
def testClassifyIncarcerationEvents(self): """Tests the ClassifyIncarcerationEvents DoFn.""" fake_person_id = 12345 fake_person = StatePerson.new_with_defaults( person_id=fake_person_id, gender=Gender.MALE, birthdate=date(1970, 1, 1), residency_status=ResidencyStatus.PERMANENT) incarceration_period = StateIncarcerationPeriod.new_with_defaults( incarceration_period_id=1111, incarceration_type=StateIncarcerationType.STATE_PRISON, status=StateIncarcerationPeriodStatus.NOT_IN_CUSTODY, state_code='TX', facility='PRISON XX', admission_date=date(2010, 11, 20), admission_reason=StateIncarcerationPeriodAdmissionReason. PROBATION_REVOCATION, release_date=date(2010, 12, 4), release_reason=StateIncarcerationPeriodReleaseReason. SENTENCE_SERVED) incarceration_sentence = StateIncarcerationSentence.new_with_defaults( incarceration_sentence_id=123, incarceration_periods=[incarceration_period], charges=[ StateCharge.new_with_defaults(ncic_code='5699', statute='CIVIL RIGHTS', offense_date=date(2009, 1, 9)) ]) sentence_group = StateSentenceGroup.new_with_defaults( sentence_group_id=123, incarceration_sentences=[incarceration_sentence]) incarceration_sentence.sentence_group = sentence_group incarceration_period.incarceration_sentences = [incarceration_sentence] person_entities = { 'person': [fake_person], 'sentence_groups': [sentence_group] } fake_person_id_to_county_query_result = [{ 'person_id': fake_person_id, 'county_of_residence': _COUNTY_OF_RESIDENCE }] incarceration_events = [ IncarcerationStayEvent( admission_reason=incarceration_period.admission_reason, admission_reason_raw_text=incarceration_period. admission_reason_raw_text, supervision_type_at_admission= StateSupervisionPeriodSupervisionType.PROBATION, state_code=incarceration_period.state_code, event_date=last_day_of_month( incarceration_period.admission_date), facility=incarceration_period.facility, county_of_residence=_COUNTY_OF_RESIDENCE, most_serious_offense_statute='CIVIL RIGHTS'), IncarcerationAdmissionEvent( state_code=incarceration_period.state_code, event_date=incarceration_period.admission_date, facility=incarceration_period.facility, county_of_residence=_COUNTY_OF_RESIDENCE, admission_reason=incarceration_period.admission_reason, admission_reason_raw_text=incarceration_period. admission_reason_raw_text, supervision_type_at_admission= StateSupervisionPeriodSupervisionType.PROBATION, ), IncarcerationReleaseEvent( state_code=incarceration_period.state_code, event_date=incarceration_period.release_date, facility=incarceration_period.facility, county_of_residence=_COUNTY_OF_RESIDENCE, release_reason=incarceration_period.release_reason) ] correct_output = [(fake_person, incarceration_events)] test_pipeline = TestPipeline() person_id_to_county_kv = ( test_pipeline | "Read person id to county associations from BigQuery" >> beam.Create(fake_person_id_to_county_query_result) | "Convert to KV" >> beam.ParDo(ConvertDictToKVTuple(), 'person_id')) output = (test_pipeline | beam.Create([(fake_person_id, person_entities)]) | 'Identify Incarceration Events' >> beam.ParDo( pipeline.ClassifyIncarcerationEvents(), AsDict(person_id_to_county_kv))) assert_that(output, equal_to(correct_output)) test_pipeline.run()
def combination_incarceration_monthly_metrics( combo: Dict[str, Any], incarceration_event: IncarcerationEvent, all_incarceration_events: List[IncarcerationEvent]) \ -> List[Tuple[Dict[str, Any], int]]: """Returns all unique incarceration metrics for the given event and combination. First, includes an event-based count for the month the event occurred with a metric period of 1 month. Then, if this event should be included in the person-based count for the month when the event occurred, adds those person- based metrics. Args: combo: A characteristic combination to convert into metrics incarceration_event: The IncarcerationEvent from which the combination was derived all_incarceration_events: All of this person's IncarcerationEvents Returns: A list of key-value tuples representing specific metric combination dictionaries and the number 1 representing a positive contribution to that count metric. """ metrics = [] event_date = incarceration_event.event_date event_year = event_date.year event_month = event_date.month # Add event-based combos for the 1-month period the month of the event event_based_same_month_combos = augmented_combo_list( combo, incarceration_event.state_code, event_year, event_month, MetricMethodologyType.EVENT, 1) for event_combo in event_based_same_month_combos: metrics.append((event_combo, 1)) # Create the person-based combos for the 1-month period of the month of the event person_based_same_month_combos = augmented_combo_list( combo, incarceration_event.state_code, event_year, event_month, MetricMethodologyType.PERSON, 1 ) events_in_month: List[IncarcerationEvent] = [] if isinstance(incarceration_event, IncarcerationAdmissionEvent): # All admission events that happened the same month as this one events_in_month = [ event for event in all_incarceration_events if isinstance(event, IncarcerationAdmissionEvent) and event.event_date.year == event_date.year and event.event_date.month == event_date.month ] elif isinstance(incarceration_event, IncarcerationStayEvent): # All stay events that happened the same month as this one events_in_month = [ event for event in all_incarceration_events if isinstance(event, IncarcerationStayEvent) and event.event_date.year == event_date.year and event.event_date.month == event_date.month ] elif isinstance(incarceration_event, IncarcerationReleaseEvent): # All release events that happened the same month as this one events_in_month = [ event for event in all_incarceration_events if isinstance(event, IncarcerationReleaseEvent) and event.event_date.year == event_date.year and event.event_date.month == event_date.month ] if events_in_month and include_event_in_count( incarceration_event, last_day_of_month(event_date), events_in_month): # Include this event in the person-based count for person_combo in person_based_same_month_combos: metrics.append((person_combo, 1)) return metrics
def find_revocation_return_buckets( supervision_sentences: List[StateSupervisionSentence], incarceration_sentences: List[StateIncarcerationSentence], supervision_periods: List[StateSupervisionPeriod], incarceration_periods: List[StateIncarcerationPeriod], supervision_time_buckets: List[SupervisionTimeBucket], assessments: List[StateAssessment], violation_responses: List[StateSupervisionViolationResponse], ssvr_agent_associations: Dict[int, Dict[Any, Any]], supervision_period_to_agent_associations: Dict[int, Dict[Any, Any]]) -> \ List[SupervisionTimeBucket]: """Looks at all incarceration periods to see if they were revocation returns. For each revocation admission, adds one RevocationReturnSupervisionTimeBuckets for each overlapping supervision period. If there are no overlapping supervision periods, looks for a recently terminated period. If there are no overlapping or recently terminated supervision periods, adds a RevocationReturnSupervisionTimeBuckets with as much information about the time on supervision as possible. """ revocation_return_buckets: List[SupervisionTimeBucket] = [] for incarceration_period in incarceration_periods: if not incarceration_period.admission_date: raise ValueError( f"Admission date for null for {incarceration_period}") if not is_revocation_admission(incarceration_period.admission_reason): continue admission_date = incarceration_period.admission_date admission_year = admission_date.year admission_month = admission_date.month end_of_month = last_day_of_month(admission_date) assessment_score, assessment_level, assessment_type = find_most_recent_assessment( end_of_month, assessments) relevant_pre_incarceration_supervision_periods = \ _get_relevant_supervision_periods_before_admission_date(admission_date, supervision_periods) if relevant_pre_incarceration_supervision_periods: # Add a RevocationReturnSupervisionTimeBucket for each overlapping supervision period for supervision_period in relevant_pre_incarceration_supervision_periods: revocation_details = _get_revocation_details( incarceration_period, supervision_period, ssvr_agent_associations, supervision_period_to_agent_associations) supervision_type_at_admission = get_pre_incarceration_supervision_type( incarceration_sentences, supervision_sentences, incarceration_period, [supervision_period]) case_type = _identify_most_severe_case_type(supervision_period) supervision_level = supervision_period.supervision_level supervision_level_raw_text = supervision_period.supervision_level_raw_text # Get details about the violation and response history leading up to the revocation violation_history = get_violation_and_response_history( admission_date, violation_responses) revocation_month_bucket = RevocationReturnSupervisionTimeBucket( state_code=incarceration_period.state_code, year=admission_year, month=admission_month, supervision_type=supervision_type_at_admission, case_type=case_type, assessment_score=assessment_score, assessment_level=assessment_level, assessment_type=assessment_type, revocation_type=revocation_details.revocation_type, source_violation_type=revocation_details. source_violation_type, most_severe_violation_type=violation_history. most_severe_violation_type, most_severe_violation_type_subtype=violation_history. most_severe_violation_type_subtype, most_severe_response_decision=violation_history. most_severe_response_decision, response_count=violation_history.response_count, violation_history_description=violation_history. violation_history_description, violation_type_frequency_counter=violation_history. violation_type_frequency_counter, supervising_officer_external_id=revocation_details. supervising_officer_external_id, supervising_district_external_id=revocation_details. supervising_district_external_id, supervision_level=supervision_level, supervision_level_raw_text=supervision_level_raw_text) revocation_return_buckets.append(revocation_month_bucket) else: # There are no overlapping or proximal supervision periods. Add one # RevocationReturnSupervisionTimeBucket with as many details as possible about this revocation revocation_details = _get_revocation_details( incarceration_period, None, ssvr_agent_associations, None) supervision_type_at_admission = get_pre_incarceration_supervision_type( incarceration_sentences, supervision_sentences, incarceration_period, []) # TODO(2853): Don't default to GENERAL once we figure out how to handle unset fields case_type = StateSupervisionCaseType.GENERAL end_of_month = last_day_of_month(admission_date) assessment_score, assessment_level, assessment_type = find_most_recent_assessment( end_of_month, assessments) # Get details about the violation and response history leading up to the revocation violation_history = get_violation_and_response_history( admission_date, violation_responses) if supervision_type_at_admission is not None: revocation_month_bucket = RevocationReturnSupervisionTimeBucket( state_code=incarceration_period.state_code, year=admission_year, month=admission_month, supervision_type=supervision_type_at_admission, case_type=case_type, assessment_score=assessment_score, assessment_level=assessment_level, assessment_type=assessment_type, revocation_type=revocation_details.revocation_type, source_violation_type=revocation_details. source_violation_type, most_severe_violation_type=violation_history. most_severe_violation_type, most_severe_violation_type_subtype=violation_history. most_severe_violation_type_subtype, most_severe_response_decision=violation_history. most_severe_response_decision, response_count=violation_history.response_count, violation_history_description=violation_history. violation_history_description, violation_type_frequency_counter=violation_history. violation_type_frequency_counter, supervising_officer_external_id=revocation_details. supervising_officer_external_id, supervising_district_external_id=revocation_details. supervising_district_external_id) revocation_return_buckets.append(revocation_month_bucket) if revocation_return_buckets: supervision_time_buckets.extend(revocation_return_buckets) return supervision_time_buckets