def _delete_empty_datasets() -> None: """Deletes all empty datasets in BigQuery.""" bq_client = BigQueryClientImpl() datasets = bq_client.list_datasets() for dataset_resource in datasets: dataset_ref = bq_client.dataset_ref_for_id(dataset_resource.dataset_id) dataset = bq_client.get_dataset(dataset_ref) tables = peekable(bq_client.list_tables(dataset.dataset_id)) created_time = dataset.created dataset_age_seconds = (datetime.datetime.now(datetime.timezone.utc) - created_time).total_seconds() if not tables and dataset_age_seconds > DATASET_DELETION_MIN_SECONDS: logging.info( "Dataset %s is empty and was not created very recently. Deleting...", dataset_ref.dataset_id, ) bq_client.delete_dataset(dataset_ref)
def main(dry_run: bool) -> None: client = BigQueryClientImpl() datasets = list(client.list_datasets()) candidate_deletable_datasets = [ d for d in datasets if d.dataset_id.startswith("temp_dataset_") ] cutoff_date = (datetime.now() - timedelta(days=1)).replace(tzinfo=pytz.UTC) for candidate in candidate_deletable_datasets: dataset = client.get_dataset(candidate.dataset_id) if dataset.modified is not None and dataset.modified < cutoff_date: if dry_run: logging.info("[Dry-run] Would delete %s", dataset.dataset_id) else: logging.info("Deleting %s...", dataset.dataset_id) client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) else: logging.info("Skipping %s because it was created too recently.", dataset.dataset_id)