def setUpClass(cls): print('**************************************************************') print(cls.__name__) print('**************************************************************') super().initialize_class_vars() # Set the test project identifier project_id = os.environ.get(PROJECT_ID) cls.project_id = project_id # Set the expected test datasets dataset_id = bq_utils.get_combined_deid_dataset_id() cls.dataset_id = dataset_id sandbox_id = dataset_id + '_sandbox' cls.sandbox_id = sandbox_id rule_instance = CleanHeightAndWeight(project_id, dataset_id, sandbox_id) cls.rule_instance = rule_instance for table_name in rule_instance.get_sandbox_tablenames(): cls.fq_sandbox_table_names.append( f'{project_id}.{sandbox_id}.{table_name}') cls.fq_table_names = [ f'{project_id}.{dataset_id}.person', f'{project_id}.{dataset_id}.measurement', f'{project_id}.{dataset_id}.measurement_ext', f'{project_id}.{dataset_id}.condition_occurrence', f'{project_id}.{dataset_id}.concept', f'{project_id}.{dataset_id}.concept_ancestor', ] super().setUpClass()
def clean_combined_de_identified_dataset(project_id=None, dataset_id=None): """ Run all clean rules defined for the deidentified ehr and rdr dataset. :param project_id: Name of the BigQuery project. :param dataset_id: Name of the dataset to clean """ if project_id is None: project_id = app_identity.get_application_id() LOGGER.info('Project is unspecified. Using default value of:\t%s', project_id) if dataset_id is None: dataset_id = bq_utils.get_combined_deid_dataset_id() LOGGER.info('Dataset is unspecified. Using default value of:\t%s', dataset_id) sandbox_dataset_id = sandbox.create_sandbox_dataset(project_id=project_id, dataset_id=dataset_id) query_list = _gather_combined_de_identified_queries( project_id, dataset_id, sandbox_dataset_id) LOGGER.info("Cleaning de-identified dataset") clean_engine.clean_dataset(project_id, query_list, stage.DEID_BASE)
def clean_ehr_rdr_de_identified_dataset(project=None, dataset=None): if dataset is None or dataset == '' or dataset.isspace(): dataset = bq_utils.get_combined_deid_dataset_id() LOGGER.info('Dataset is unspecified. Using default value of:\t%s', dataset) query_list = _gather_ehr_rdr_de_identified_queries(project, dataset) LOGGER.info("Cleaning de-identified dataset") clean_engine.clean_dataset(project, dataset, query_list)
def get_dataset_and_project_names(): """ Get project and dataset names from environment variables. :return: A dictionary of dataset names and project name """ project_and_dataset_names = dict() project_and_dataset_names[clean_cdr_consts.EHR_DATASET] = bq_utils.get_dataset_id() project_and_dataset_names[clean_cdr_consts.UNIONED_EHR_DATASET] = bq_utils.get_unioned_dataset_id() project_and_dataset_names[clean_cdr_consts.RDR_DATASET] = bq_utils.get_rdr_dataset_id() project_and_dataset_names[clean_cdr_consts.EHR_RDR_DATASET] = bq_utils.get_ehr_rdr_dataset_id() project_and_dataset_names[clean_cdr_consts.EHR_RDR_DE_IDENTIFIED] = bq_utils.get_combined_deid_dataset_id() project_and_dataset_names[clean_cdr_consts.PROJECT] = app_identity.get_application_id() return project_and_dataset_names
def clean_ehr_rdr_de_identified_dataset(project_id=None, dataset_id=None): """ Run all clean rules defined for the deidentified ehr and rdr dataset. :param project_id: Name of the BigQuery project. :param dataset_id: Name of the dataset to clean """ if dataset_id is None or dataset_id == '' or dataset_id.isspace(): dataset_id = bq_utils.get_combined_deid_dataset_id() LOGGER.info('Dataset is unspecified. Using default value of:\t%s', dataset_id) query_list = _gather_ehr_rdr_de_identified_queries(project_id, dataset_id) LOGGER.info("Cleaning de-identified dataset") clean_engine.clean_dataset(project_id, dataset_id, query_list)