def check_vehicle_accident_suppression(check_df, project_id, post_deid_dataset, pre_deid_dataset=None, mapping_dataset=None): """Run motor vehicle accident suppression check Parameters ---------- check_df: pd.DataFrame Dataframe containing the checks that need to be done project_id: str Google Bigquery project_id post_dataset_id: str Bigquery dataset after de-id rules were run pre_deid_dataset: str Bigquery dataset before de-id rules were run Returns ------- pd.DataFrame """ icd9_vehicle_accident = run_check_by_row( check_df, QUERY_VEHICLE_ACCIDENT_SUPPRESSION_ICD9, project_id, post_deid_dataset) icd10_vehicle_accident = run_check_by_row( check_df, QUERY_VEHICLE_ACCIDENT_SUPPRESSION_ICD10, project_id, post_deid_dataset) return pd.concat([icd9_vehicle_accident, icd10_vehicle_accident], sort=True)
def check_mapping(check_df, project_id, post_dataset_id, pre_deid_dataset, mapping_dataset): """Run mapping verification rules Parameters ---------- check_df: pd.DataFrame Dataframe containing the checks that need to be done project_id: str Google Bigquery project_id post_dataset_id: str Bigquery dataset after de-id rules were run pre_deid_dataset: str Bigquery dataset before de-id rules were run Returns ------- pd.DataFrame """ # Correct type type_check = run_check_by_row(check_df, QUERY_ID_NOT_OF_CORRECT_TYPE, project_id, post_dataset_id, pre_deid_dataset, "ID datatype has been changed", mapping_dataset) # id changed by de-id id_map_check = run_check_by_row(check_df, QUERY_ID_NOT_CHANGED_BY_DEID, project_id, post_dataset_id, pre_deid_dataset, "Old ID still in the dataset", mapping_dataset) # new id not in mapping id_not_in_mapping_check = run_check_by_row( check_df, QUERY_ID_NOT_IN_MAPPING, project_id, post_dataset_id, pre_deid_dataset, "New ID not found in mapping table", mapping_dataset) # # new id properly mapped to old id id_properly_mapped_check = run_check_by_row( check_df, QUERY_ID_NOT_MAPPED_PROPERLY, project_id, post_dataset_id, pre_deid_dataset, "ID not properly mapped", mapping_dataset) return pd.concat([ type_check, id_map_check, id_not_in_mapping_check, id_properly_mapped_check ], sort=True)
def check_field_cancer_concept_suppression(check_df, project_id, post_deid_dataset, pre_deid_dataset=None, mapping_dataset=None): """Run suppression check for some cancer concepts Parameters ---------- check_df: pd.DataFrame Dataframe containing the checks that need to be done project_id: str Google Bigquery project_id post_dataset_id: str Bigquery dataset after de-id rules were run pre_deid_dataset: str Bigquery dataset before de-id rules were run Returns ------- pd.DataFrame """ cancer_concept = run_check_by_row(check_df, QUERY_CANCER_CONCEPT_SUPPRESSION, project_id, post_deid_dataset) return cancer_concept
def check_field_geolocation_records_suppression(check_df, project_id, post_deid_dataset, pre_deid_dataset=None, mapping_dataset=None): return run_check_by_row(check_df, QUERY_GEOLOCATION_SUPPRESSION, project_id, post_deid_dataset)
def check_table_suppression(check_df, project_id, post_dataset_id, pre_deid_dataset=None, mapping_dataset=None): """Run table suppression check Parameters ---------- check_df: pd.DataFrame Dataframe containing the checks that need to be done project_id: str Google Bigquery project_id post_dataset_id: str Bigquery dataset after de-id rules were run pre_deid_dataset: str Bigquery dataset before de-id rules were run Returns ------- pd.DataFrame """ table_check = run_check_by_row(check_df, QUERY_SUPPRESSED_TABLE, project_id, post_dataset_id) return table_check.reset_index(drop=True)
def check_field_suppression(check_df, project_id, post_dataset_id, pre_deid_dataset=None, mapping_dataset=None): """Run field suppression check Parameters ---------- check_df: pd.DataFrame Dataframe containing the checks that need to be done project_id: str Google Bigquery project_id post_dataset_id: str Bigquery dataset after de-id rules were run pre_deid_dataset: str Bigquery dataset before de-id rules were run Returns ------- pd.DataFrame """ nullable_field = check_df[check_df['is_nullable'] == 'YES'] required_numeric_field = check_df[(check_df['is_nullable'] == 'NO') & (check_df['data_type'] == 'INT64')] required_other_field = check_df[(check_df['is_nullable'] == 'NO') & (check_df['data_type'] != 'INT64')] nullable_field_check = run_check_by_row( nullable_field, QUERY_SUPPRESSED_NULLABLE_FIELD_NOT_NULL, project_id, post_dataset_id) required_numeric_field_check = run_check_by_row( required_numeric_field, QUERY_SUPPRESSED_NUMERIC_NOT_ZERO, project_id, post_dataset_id) required_other_field_check = run_check_by_row( required_other_field, QUERY_SUPPRESSED_REQUIRED_FIELD_NOT_EMPTY, project_id, post_dataset_id) return pd.concat([ nullable_field_check, required_numeric_field_check, required_other_field_check ], sort=True)
def check_field_freetext_response_suppression(check_df, project_id, post_deid_dataset, pre_deid_dataset=None, mapping_dataset=None): free_text_concept = run_check_by_row(check_df, QUERY_SUPPRESSED_FREE_TEXT_RESPONSE, project_id, post_deid_dataset) return free_text_concept
def check_mapping_zipcode_generalization(check_df, project_id, post_dataset_id, pre_deid_dataset, mapping_dataset): """Run zipcode generalization and transformation check Parameters ---------- check_df: pd.DataFrame Dataframe containing the checks that need to be done project_id: str Google Bigquery project_id post_dataset_id: str Bigquery dataset after de-id rules were run pre_deid_dataset: str Bigquery dataset before de-id rules were run Returns ------- pd.DataFrame """ zip_check = run_check_by_row(check_df, QUERY_ZIP_CODE_GENERALIZATION, project_id, post_dataset_id, pre_deid_dataset, "Zip code value generalized and transformed") return zip_check