def make_dataset(project, dataset_id, friendly_name=None, description=None, default_table_expiration_ms=None, location=None, labels=None, access_entries=None): dataset_ref = DatasetReference(project, dataset_id) dataset = Dataset(dataset_ref) dataset.friendly_name = friendly_name dataset.description = description dataset.default_table_expiration_ms = default_table_expiration_ms dataset.location = location if labels is not None: dataset.labels = labels if access_entries is not None: dataset.access_entries = access_entries return dataset
def create_scratch_dataset( context: AbstractComputeExecutionContext) -> HcaScratchDatasetName: """ Creates a staging dataset that will house records for update/insertion into the final TDR dataset :return: Name of the staging dataset """ scratch_bq_project = context.resources.scratch_config.scratch_bq_project scratch_dataset_prefix = context.resources.scratch_config.scratch_dataset_prefix load_tag = context.resources.load_tag dataset_name = f"{scratch_bq_project}.{scratch_dataset_prefix}_{load_tag}" dataset = Dataset(dataset_name) dataset.default_table_expiration_ms = context.resources.scratch_config.scratch_table_expiration_ms bq_client = context.resources.bigquery_client bq_client.create_dataset(dataset) context.log.info(f"Created scratch dataset {dataset_name}") return HcaScratchDatasetName(dataset_name)
def __dataset(cls): dataset = Dataset(DatasetReference(GCP_PROJECT_ID, "it_test_dataset")) dataset.default_table_expiration_ms = cls.ONE_DAY_IN_MS return dataset