def test_load_bucket_data_into_bigquery_external_table_using_csv_data(
self):
gcp_interfacer.authenticate_gcp(
credentials_file_path=self.credentials_file_path)
gcp_interfacer.create_gcp_storage_bucket(bucket_name=self.bucket_name)
destination_blob_name = "user_data.csv"
source_file_name = "/".join([
os.path.abspath("integration_tests/resources/csv_data/"),
destination_blob_name
])
gcp_interfacer.upload_blob(bucket_name=self.bucket_name,
source_file_name=source_file_name,
destination_blob_name=destination_blob_name)
gcp_interfacer.create_bigquery_dataset(
bigquery_client=self.bigquery_client,
dataset_name=self.dataset_name,
dataset_location="US")
actual = gcp_interfacer.load_bucket_data_into_bigquery_external_table(
bigquery_client=self.bigquery_client,
dataset_name=self.dataset_name,
bucket_name=self.bucket_name,
table_id=self.table_name,
blob_prefix=destination_blob_name[0:3])
self.assertIsNone(actual)
gcp_interfacer.delete_blob(bucket_name=self.bucket_name,
blob_name=destination_blob_name)
gcp_interfacer.delete_gcp_storage_bucket(bucket_name=self.bucket_name)
gcp_interfacer.delete_bigquery_dataset(
bigquery_client=self.bigquery_client,
dataset_name=self.dataset_name)
def test_delete_blob(self):
gcp_interfacer.create_gcp_storage_bucket(bucket_name=self.bucket_name)
destination_blob_name = "user_data.csv"
source_file_name = "/".join([
os.path.abspath("integration_tests/resources/csv_data/"),
destination_blob_name
])
gcp_interfacer.upload_blob(bucket_name=self.bucket_name,
source_file_name=source_file_name,
destination_blob_name=destination_blob_name)
actual = gcp_interfacer.delete_blob(bucket_name=self.bucket_name,
blob_name=destination_blob_name)
gcp_interfacer.delete_gcp_storage_bucket(bucket_name=self.bucket_name)
self.assertIsNone(actual)
def test_list_blobs(self):
gcp_interfacer.create_gcp_storage_bucket(bucket_name=self.bucket_name)
destination_blob_name = "user_data.csv"
source_file_name = "/".join([
os.path.abspath("integration_tests/resources/csv_data/"),
destination_blob_name
])
gcp_interfacer.upload_blob(bucket_name=self.bucket_name,
source_file_name=source_file_name,
destination_blob_name=destination_blob_name)
actual = gcp_interfacer.get_list_of_blobs(
bucket_name=self.bucket_name,
blob_prefix=destination_blob_name[0:3])
gcp_interfacer.delete_blob(bucket_name=self.bucket_name,
blob_name=destination_blob_name)
gcp_interfacer.delete_gcp_storage_bucket(bucket_name=self.bucket_name)
expected = [destination_blob_name]
self.assertEqual(expected, actual)
def test_delete_gcp_storage_bucket(self):
gcp_interfacer.create_gcp_storage_bucket(bucket_name=self.bucket_name)
gcp_interfacer.delete_gcp_storage_bucket(bucket_name=self.bucket_name)
storage_client = storage.Client()
self.assertRaises(google.api_core.exceptions.NotFound,
storage_client.get_bucket, self.bucket_name)
def test_create_gcp_storage_bucket(self):
gcp_interfacer.create_gcp_storage_bucket(bucket_name=self.bucket_name)
storage_client = storage.Client()
actual = storage_client.get_bucket(bucket_or_name=self.bucket_name)
gcp_interfacer.delete_gcp_storage_bucket(bucket_name=self.bucket_name)
self.assertIsNotNone(actual)