def make_dataset(project,
dataset_id,
friendly_name=None,
description=None,
default_table_expiration_ms=None,
location=None,
labels=None,
access_entries=None):
dataset_ref = DatasetReference(project, dataset_id)
dataset = Dataset(dataset_ref)
dataset.friendly_name = friendly_name
dataset.description = description
dataset.default_table_expiration_ms = default_table_expiration_ms
dataset.location = location
if labels is not None:
dataset.labels = labels
if access_entries is not None:
dataset.access_entries = access_entries
return dataset
def create_scratch_dataset(
context: AbstractComputeExecutionContext) -> HcaScratchDatasetName:
"""
Creates a staging dataset that will house records for update/insertion into the
final TDR dataset
:return: Name of the staging dataset
"""
scratch_bq_project = context.resources.scratch_config.scratch_bq_project
scratch_dataset_prefix = context.resources.scratch_config.scratch_dataset_prefix
load_tag = context.resources.load_tag
dataset_name = f"{scratch_bq_project}.{scratch_dataset_prefix}_{load_tag}"
dataset = Dataset(dataset_name)
dataset.default_table_expiration_ms = context.resources.scratch_config.scratch_table_expiration_ms
bq_client = context.resources.bigquery_client
bq_client.create_dataset(dataset)
context.log.info(f"Created scratch dataset {dataset_name}")
return HcaScratchDatasetName(dataset_name)
def __dataset(cls):
dataset = Dataset(DatasetReference(GCP_PROJECT_ID, "it_test_dataset"))
dataset.default_table_expiration_ms = cls.ONE_DAY_IN_MS
return dataset
