def __init__(self,
ingest_directory_path: Optional[str] = None,
storage_directory_path: Optional[str] = None,
max_delay_sec_between_files: Optional[int] = None):
super(UsPaController, self).__init__(
'us_pa',
SystemLevel.STATE,
ingest_directory_path,
storage_directory_path,
max_delay_sec_between_files=max_delay_sec_between_files)
self.enum_overrides = self.generate_enum_overrides()
self.row_post_processors_by_file: Dict[str, List[Callable]] = {
'dbo_IcsDoc': [
gen_label_single_external_id_hook(US_PA_SID),
self.gen_hydrate_alternate_external_ids({
'Cont_Num': US_PA_CONTROL,
'PBPP_Num': US_PA_PBPP,
}),
copy_name_to_alias,
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
self._compose_current_address,
],
'dbo_tblInmTestScore': [
self.gen_hydrate_alternate_external_ids({
'Control_Number':
US_PA_CONTROL,
}),
self._generate_doc_assessment_external_id,
self._enrich_doc_assessments,
],
'dbo_Senrec': [
self._set_incarceration_sentence_id,
self._enrich_incarceration_sentence,
self._strip_id_whitespace,
gen_set_agent_type(StateAgentType.JUDGE),
],
'dbo_Offender': [
gen_label_single_external_id_hook(US_PA_PBPP),
self.gen_hydrate_alternate_external_ids({
'OffSID': US_PA_SID,
}),
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
],
'dbo_LSIR': [
gen_label_single_external_id_hook(US_PA_PBPP),
self._generate_pbpp_assessment_external_id,
self._enrich_pbpp_assessments,
]
}
self.file_post_processors_by_file: Dict[str, List[Callable]] = {
'dbo_IcsDoc': [],
'dbo_tblInmTestScore': [],
'dbo_Senrec': [],
'dbo_Offender': [],
'dbo_LSIR': [],
}
def __init__(self,
ingest_directory_path: Optional[str] = None,
storage_directory_path: Optional[str] = None,
max_delay_sec_between_files: Optional[int] = None):
super(UsPaController, self).__init__(
'us_pa',
SystemLevel.STATE,
ingest_directory_path,
storage_directory_path,
max_delay_sec_between_files=max_delay_sec_between_files)
self.enum_overrides = self.generate_enum_overrides()
self.row_post_processors_by_file: Dict[str, List[Callable]] = {
'dbo_IcsDoc': [
gen_label_single_external_id_hook(US_PA_DOC),
self.gen_hydrate_alternate_external_ids({
'Cont_Num': US_PA_CONT,
'Inmate_Num': US_PA_INMATE,
'PBPP_Num': US_PA_PBPP,
'SID_Num': US_PA_SID,
}),
copy_name_to_alias,
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
self._compose_current_address,
],
'dbo_Offender': [
gen_label_single_external_id_hook(US_PA_PBPP),
self.gen_hydrate_alternate_external_ids({
'OffSID': US_PA_SID,
}),
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
],
}
self.file_post_processors_by_file: Dict[str, List[Callable]] = {
'dbo_IcsDoc': [],
'dbo_Offender': [],
}
def __init__(self,
ingest_directory_path: Optional[str] = None,
storage_directory_path: Optional[str] = None,
max_delay_sec_between_files: Optional[int] = None):
super(UsIdController, self).__init__(
'us_id',
SystemLevel.STATE,
ingest_directory_path,
storage_directory_path,
max_delay_sec_between_files=max_delay_sec_between_files)
self.enum_overrides = self.generate_enum_overrides()
self.row_post_processors_by_file: Dict[str, List[Callable]] = {
'offender_ofndr_dob_address': [
copy_name_to_alias,
# When first parsed, the info object just has a single external id - the DOC id.
gen_label_single_external_id_hook(US_ID_DOC),
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
],
'ofndr_tst_ofndr_tst_cert': [
self._add_lsir_to_assessments,
gen_label_single_external_id_hook(US_ID_DOC),
],
'mittimus_judge_sentence_offense_sentprob_incarceration_sentences':
[
gen_label_single_external_id_hook(US_ID_DOC),
self._add_statute_to_charge,
self._add_judge_to_court_cases,
self._set_extra_sentence_fields,
self._set_generated_ids,
],
'mittimus_judge_sentence_offense_sentprob_supervision_sentences': [
gen_label_single_external_id_hook(US_ID_DOC),
self._add_statute_to_charge,
self._add_judge_to_court_cases,
self._set_extra_sentence_fields,
self._set_generated_ids,
],
'early_discharge_incarceration_sentence': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
],
'early_discharge_supervision_sentence': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
],
'movement_facility_location_offstat_incarceration_periods': [
gen_label_single_external_id_hook(US_ID_DOC),
self._override_facilities,
self._set_generated_ids,
self._clear_max_dates,
self._add_incarceration_type,
self._add_default_admission_reason,
],
'movement_facility_location_offstat_supervision_periods': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
self._clear_max_dates,
self._supervision_period_admission_and_termination_overrides,
self._add_default_admission_reason,
self._override_supervision_fields_from_location_info,
self._set_case_type_from_supervision_level,
self._add_supervising_officer,
],
'ofndr_tst_tst_qstn_rspns_violation_reports': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
self._set_violation_violent_sex_offense,
self._hydrate_violation_types,
self._hydrate_violation_report_fields,
],
'ofndr_tst_tst_qstn_rspns_violation_reports_old': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
self._set_violation_violent_sex_offense,
self._hydrate_violation_types,
self._hydrate_violation_report_fields,
],
'sprvsn_cntc': [
gen_label_single_external_id_hook(US_ID_DOC),
self._add_supervision_contact_fields,
],
}
self.file_post_processors_by_file: Dict[str, List[Callable]] = {
'offender_ofndr_dob_address': [],
'ofndr_tst_ofndr_tst_cert': [],
'mittimus_judge_sentence_offense_sentprob_incarceration_sentences':
[],
'mittimus_judge_sentence_offense_sentprob_supervision_sentences':
[],
'early_discharge_incarceration_sentence': [],
'early_discharge_supervision_sentence': [],
'movement_facility_location_offstat_incarceration_periods': [],
'movement_facility_location_offstat_supervision_periods': [],
'ofndr_tst_tst_qstn_rspns_violation_reports': [],
'ofndr_tst_tst_qstn_rspns_violation_reports_old': [],
'sprvsn_cntc': [],
}
def __init__(self,
ingest_directory_path: Optional[str] = None,
storage_directory_path: Optional[str] = None,
max_delay_sec_between_files: Optional[int] = None):
super(UsPaController, self).__init__(
'us_pa',
SystemLevel.STATE,
ingest_directory_path,
storage_directory_path,
max_delay_sec_between_files=max_delay_sec_between_files)
self.enum_overrides = self.generate_enum_overrides()
self.row_post_processors_by_file: Dict[str, List[Callable]] = {
'person_external_ids': [self._hydrate_person_external_ids],
'doc_person_info': [
gen_label_single_external_id_hook(US_PA_CONTROL),
self.gen_hydrate_alternate_external_ids({
'SID_Num': US_PA_SID,
'PBPP_Num': US_PA_PBPP,
}), copy_name_to_alias,
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
self._compose_current_address, self._hydrate_sentence_group_ids
],
'dbo_tblInmTestScore': [
gen_label_single_external_id_hook(US_PA_CONTROL),
self._generate_doc_assessment_external_id,
self._enrich_doc_assessments,
],
'dbo_Senrec': [
self._set_incarceration_sentence_id,
self._enrich_incarceration_sentence,
self._strip_id_whitespace,
gen_set_agent_type(StateAgentType.JUDGE),
],
'incarceration_period': [
gen_label_single_external_id_hook(US_PA_CONTROL),
self._concatenate_admission_reason_codes,
self._concatenate_release_reason_codes,
self._concatenate_incarceration_purpose_codes,
self._add_incarceration_type,
],
'dbo_Miscon': [
gen_label_single_external_id_hook(US_PA_CONTROL),
self._specify_incident_location,
self._specify_incident_type,
self._specify_incident_details,
self._specify_incident_outcome,
],
'dbo_Offender': [
gen_label_single_external_id_hook(US_PA_PBPP),
self._hydrate_races,
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
],
'dbo_LSIR': [
gen_label_single_external_id_hook(US_PA_PBPP),
self._generate_pbpp_assessment_external_id,
self._enrich_pbpp_assessments,
],
'supervision_sentence': [
self._enrich_supervision_sentence,
self._set_default_supervision_sentence_type,
self._append_additional_charges_to_supervision_sentence,
],
}
self.file_post_processors_by_file: Dict[str, List[Callable]] = {
'person_external_ids': [],
'doc_person_info': [],
'dbo_tblInmTestScore': [],
'dbo_Senrec': [
gen_convert_person_ids_to_external_id_objects(
self._get_id_type),
],
'incarceration_period': [
gen_convert_person_ids_to_external_id_objects(
self._get_id_type),
],
'dbo_Miscon': [
gen_convert_person_ids_to_external_id_objects(
self._get_id_type),
],
'dbo_Offender': [],
'dbo_LSIR': [],
'supervision_sentence': [
gen_convert_person_ids_to_external_id_objects(
self._get_id_type),
],
}
self.primary_key_override_hook_by_file: Dict[str, Callable] = {
'incarceration_period': _generate_incarceration_period_primary_key,
'supervision_sentence': _generate_supervision_sentence_primary_key,
}
self.ancestor_chain_overrides_callback_by_file: Dict[str, Callable] = {
'incarceration_period':
_state_incarceration_period_ancestor_chain_overrides,
'supervision_sentence':
_state_supervision_sentence_ancestor_chain_overrides,
}
def __init__(self,
ingest_directory_path: Optional[str] = None,
storage_directory_path: Optional[str] = None,
max_delay_sec_between_files: Optional[int] = None):
super(UsNdController, self).__init__(
'us_nd',
SystemLevel.STATE,
ingest_directory_path,
storage_directory_path,
max_delay_sec_between_files=max_delay_sec_between_files)
self.row_pre_processors_by_file: Dict[str, List[Callable]] = {
'elite_offenderidentifier': [self._normalize_id_fields],
'elite_alias': [self._normalize_id_fields],
'elite_offendersentences': [self._normalize_id_fields],
'elite_offendersentenceterms': [self._normalize_id_fields],
'elite_offenderchargestable': [self._normalize_id_fields],
'elite_orderstable': [self._normalize_id_fields],
'elite_externalmovements': [self._normalize_id_fields],
}
self.row_post_processors_by_file: Dict[str, List[Callable]] = {
'elite_offenders': [copy_name_to_alias],
'elite_offenderidentifier': [self._normalize_external_id_type],
'elite_offendersentenceaggs': [
gen_set_is_life_sentence_hook('MAX_TERM', 'LIFE', StateSentenceGroup),
self._rationalize_max_length
],
'elite_offendersentences': [
gen_set_is_life_sentence_hook('SENTENCE_CALC_TYPE', 'LIFE', StateIncarcerationSentence)
],
'elite_offendersentenceterms': [self._add_sentence_children],
'elite_offenderchargestable': [
self._parse_elite_charge_classification,
self._set_elite_charge_status,
self._rationalize_controlling_charge
],
'elite_orderstable': [
self._set_judge_agent_type,
gen_normalize_county_codes_posthook(
self.region.region_code, 'COUNTY_CODE', StateCourtCase, normalized_county_code),
self._normalize_judicial_district_code
],
'elite_externalmovements': [self._process_external_movement],
'elite_offense_in_custody_and_pos_report_data': [
self._rationalize_incident_type,
self._rationalize_outcome_type,
self._set_punishment_length_days,
self._set_location_within_facility,
self._set_incident_outcome_id
],
'docstars_offenders': [
# For a person we are seeing in Docstars with an ITAGROOT_ID referencing a corresponding record in
# Elite, mark that external id as having type ELITE.
gen_label_single_external_id_hook(US_ND_ELITE),
self._enrich_addresses,
self._enrich_sorac_assessments,
copy_name_to_alias,
self._add_supervising_officer,
self._add_case_type_external_id
],
'docstars_offendercasestable': [
self._concatenate_docstars_length_periods,
self._add_terminating_officer_to_supervision_periods,
self._record_revocation,
self._set_judge_agent_type,
self._hydrate_supervision_period_sentence_shared_date_fields,
gen_normalize_county_codes_posthook(
self.region.region_code, 'TB_CTY', StateSupervisionPeriod, normalized_county_code),
],
'docstars_offensestable': [
self._parse_docstars_charge_classification,
gen_normalize_county_codes_posthook(
self.region.region_code, 'COUNTY', StateCharge, normalized_county_code),
],
'docstars_lsi_chronology': [self._process_lsir_assessments],
'docstars_ftr_episode': [self._process_ftr_episode],
}
self.primary_key_override_hook_by_file: Dict[str, Callable] = {
'elite_offendersentences': _generate_sentence_primary_key,
'elite_externalmovements': _generate_period_primary_key,
'elite_offenderchargestable': _generate_charge_primary_key,
'elite_offense_in_custody_and_pos_report_data': _generate_incident_primary_key,
}
self.ancestor_chain_overrides_callback_by_file: Dict[str, Callable] = {
'elite_offenderchargestable': _state_charge_ancestor_chain_overrides
}
self.files_to_set_with_empty_values = [
'elite_offendersentenceterms'
]
def __init__(self,
ingest_directory_path: Optional[str] = None,
storage_directory_path: Optional[str] = None,
max_delay_sec_between_files: Optional[int] = None):
super(UsIdController, self).__init__(
'us_id',
SystemLevel.STATE,
ingest_directory_path,
storage_directory_path,
max_delay_sec_between_files=max_delay_sec_between_files)
self.enum_overrides = self.generate_enum_overrides()
self.row_post_processors_by_file: Dict[str, List[Callable]] = {
'offender_ofndr_dob': [
copy_name_to_alias,
# When first parsed, the info object just has a single external id - the DOC id.
gen_label_single_external_id_hook(US_ID_DOC),
gen_rationalize_race_and_ethnicity(self.ENUM_OVERRIDES),
],
'ofndr_tst_ofndr_tst_cert': [
self._add_lsir_to_assessments,
gen_label_single_external_id_hook(US_ID_DOC),
],
'mittimus_judge_sentence_offense_sentprob_incarceration_sentences':
[
gen_label_single_external_id_hook(US_ID_DOC),
self._add_statute_to_charge,
self._add_judge_to_court_cases,
self._set_extra_sentence_fields,
self._set_generated_ids,
],
'mittimus_judge_sentence_offense_sentprob_supervision_sentences': [
gen_label_single_external_id_hook(US_ID_DOC),
self._add_statute_to_charge,
self._add_judge_to_court_cases,
self._set_extra_sentence_fields,
self._set_generated_ids,
],
'early_discharge_incarceration_sentence': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
],
'early_discharge_supervision_sentence': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
],
'movement_facility_location_offstat_incarceration_periods': [
gen_label_single_external_id_hook(US_ID_DOC),
self._override_facilities,
self._set_generated_ids,
self._clear_max_dates,
self._add_incarceration_type,
self._add_default_admission_reason,
],
'movement_facility_location_offstat_supervision_periods': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids, self._clear_max_dates,
self._supervision_period_admission_and_termination_overrides,
self._add_default_admission_reason,
self._override_supervision_fields_from_location_info
],
'ofndr_tst_tst_qstn_rspns_violation_reports': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
self._set_violation_violent_sex_offense,
self._hydrate_violation_types,
self._hydrate_violation_report_fields,
],
'ofndr_tst_tst_qstn_rspns_violation_reports_old': [
gen_label_single_external_id_hook(US_ID_DOC),
self._set_generated_ids,
self._set_violation_violent_sex_offense,
self._hydrate_violation_types,
self._hydrate_violation_report_fields,
],
'ofndr_agnt_applc_usr_body_loc_cd_current_pos': [
gen_label_single_external_id_hook(US_ID_DOC),
self._add_supervising_officer,
],
}
self.file_post_processors_by_file: Dict[str, List[Callable]] = {
'offender_ofndr_dob': [],
'ofndr_tst_ofndr_tst_cert': [],
'mittimus_judge_sentence_offense_sentprob_incarceration_sentences':
[],
'mittimus_judge_sentence_offense_sentprob_supervision_sentences':
[],
'early_discharge_incarceration_sentence': [],
'early_discharge_supervision_sentence': [],
'movement_facility_location_offstat_incarceration_periods': [],
'movement_facility_location_offstat_supervision_periods': [],
'ofndr_tst_tst_qstn_rspns_violation_reports': [],
'ofndr_tst_tst_qstn_rspns_violation_reports_old': [],
'ofndr_agnt_applc_usr_body_loc_cd_current_pos': [
# TODO(1883): Would not need this file postprocessor if our data extractor would add the main entity to
# our 'extracted_objects' cache when a single id field is used in the following situation:
# - the only key in "keys"
# - reused as a child key
gen_convert_person_ids_to_external_id_objects(self._get_id_type
),
],
}
