def test_get_dataset_name(self):
"""Test _get_dataset_name helper."""
project_id = FAKE.slug()
dataset_name = FAKE.slug()
datasets = [f"{project_id}:{dataset_name}", dataset_name]
for dataset in datasets:
billing_source = {"table_id": FAKE.slug(), "dataset": dataset}
credentials = {"project_id": project_id}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag",
return_value=self.etag,
):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
):
downloader = GCPReportDownloader(
customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials,
)
self.assertEqual(downloader._get_dataset_name(), dataset_name)
def test_init_unreachable_bucket_raises_error(self, mock_provider):
"""Assert GCPReportDownloader raises error when bucket is not reachable."""
mock_provider.return_value.cost_usage_source_is_reachable.side_effect = ValidationError
with self.assertRaises(GCPReportDownloaderError):
GCPReportDownloader(
Mock(request=Mock(id=str(FAKE.uuid4()), return_value={})),
FAKE.name(), {"bucket": FAKE.slug()})
def test_download_with_unreachable_source(self, gcp_provider):
"""Assert errors correctly when source is unreachable."""
gcp_provider.return_value.cost_usage_source_is_reachable.side_effect = ValidationError
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
with self.assertRaises(GCPReportDownloaderError):
GCPReportDownloader(FAKE.name(), billing_source, credentials=credentials)
def test_generate_etag(self, gcp_provider):
"""Test BigQuery client is handled correctly in generate etag method."""
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
with patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery") as bigquery:
bigquery.Client.return_value.get_table.return_value.modified.return_value = self.today
downloader = GCPReportDownloader(
customer_name=FAKE.name(), data_source=billing_source, provider_uuid=uuid4(), credentials=credentials
)
self.assertIsNotNone(downloader.etag)
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == AMAZON_WEB_SERVICES:
return AWSReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AWS_LOCAL_SERVICE_PROVIDER:
return AWSLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE:
return AzureReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE_LOCAL_SERVICE_PROVIDER:
return AzureLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == OPENSHIFT_CONTAINER_PLATFORM:
return OCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == GCP:
return GCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
return None
def test_generate_etag_big_query_client_error(self, gcp_provider):
"""Test BigQuery client is handled correctly in generate etag method."""
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
err_msg = "GCP Error"
with patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery") as bigquery:
bigquery.Client.side_effect = GoogleCloudError(err_msg)
with self.assertRaisesRegexp(GCPReportDownloaderError, err_msg):
GCPReportDownloader(
customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials,
)
def test_init_reachable_bucket_is_okay(self):
"""Assert GCPReportDownloader initializes with expected values."""
customer_name = FAKE.name()
bucket_name = FAKE.slug()
billing_source = {"bucket": bucket_name}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch("masu.external.downloader.gcp.gcp_report_downloader.storage"
) as mock_storage:
downloader = GCPReportDownloader(customer_name, billing_source)
mock_storage.Client.return_value.lookup_bucket.assert_called_with(
bucket_name)
self.assertEqual(downloader.customer_name,
customer_name.replace(" ", "_"))
self.assertEqual(downloader.bucket_name, bucket_name)
def test_init_reachable_bucket_is_okay(self):
"""Assert GCPReportDownloader initializes with expected values."""
customer_name = FAKE.name()
bucket_name = FAKE.slug()
billing_source = {'bucket': bucket_name}
with patch(
'masu.external.downloader.gcp.gcp_report_downloader.GCPProvider'
) as mock_provider, patch(
'masu.external.downloader.gcp.gcp_report_downloader.storage'
) as mock_storage:
downloader = GCPReportDownloader(Mock(request=Mock(id=str(FAKE.uuid4()),
return_value={})),
customer_name, billing_source)
mock_storage.Client.return_value.lookup_bucket.assert_called_with(
bucket_name
)
self.assertEqual(downloader.customer_name, customer_name.replace(' ', '_'))
self.assertEqual(downloader.bucket_name, bucket_name)
def create_gcp_downloader_with_mock_gcp_storage(
self, customer_name=None, bucket_name=None, provider_uuid=None, report_prefix=None
):
"""
Create a GCPReportDownloader instance that skips the initial GCP client/bucket check.
This also results in Mock objects being set to instance variables that can be patched
inside other test functions.
Args:
customer_name (str): optional customer name; will be randomly generated if None
bucket_name (str): optional bucket name; will be randomly generated if None
provider_uuid (uuid): optional provider UUID; will be randomly generated if None
report_prefix (str): optional report prefix
Returns:
GCPReportDownloader instance with faked argument data and Mocks in
self._storage_client and self._bucket_info.
"""
if not customer_name:
customer_name = FAKE.name()
if not bucket_name:
bucket_name = FAKE.slug()
billing_source = {'bucket': bucket_name}
if report_prefix:
billing_source['report_prefix'] = report_prefix
if not provider_uuid:
provider_uuid = uuid4()
with patch(
'masu.external.downloader.gcp.gcp_report_downloader.GCPProvider'
) as mock_provider, patch(
'masu.external.downloader.gcp.gcp_report_downloader.storage'
) as mock_storage:
# mock_storage_client = mock_storage.Client.return_value
# mock_storage_client.lookup_bucket.return_value = {}
downloader = GCPReportDownloader(
task=Mock(request=Mock(id=str(FAKE.uuid4()),
return_value={})),
customer_name=customer_name,
billing_source=billing_source,
provider_uuid=provider_uuid,
)
return downloader
def create_gcp_downloader_with_mocked_values(
self,
customer_name=FAKE.name(),
dataset=FAKE.slug(),
provider_uuid=uuid4(),
project_id=FAKE.slug(),
table_id=FAKE.slug(),
):
"""
Create a GCPReportDownloader that skips the initial GCP bigquery check creates etag.
This also results in Mock objects being set to instance variables that can be patched
inside other test functions.
Args:
customer_name (str): optional customer name; will be randomly generated if None
bucket_name (str): optional bucket name; will be randomly generated if None
provider_uuid (uuid): optional provider UUID; will be randomly generated if None
Returns:
GCPReportDownloader instance with faked argument data and Mocks in
self.etag.
"""
billing_source = {"table_id": table_id, "dataset": dataset}
credentials = {"project_id": project_id}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag",
return_value=self.etag,
):
downloader = GCPReportDownloader(
customer_name=customer_name,
data_source=billing_source,
provider_uuid=provider_uuid,
credentials=credentials,
)
return downloader
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == Provider.PROVIDER_AWS:
return AWSReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AWS_LOCAL:
return AWSLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE:
return AzureReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE_LOCAL:
return AzureLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_OCP:
return OCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_GCP:
return GCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
return None
def test_init_unreachable_bucket_raises_error(self, mock_provider):
"""Assert GCPReportDownloader raises error when bucket is not reachable."""
mock_provider.return_value.cost_usage_source_is_reachable.side_effect = (
ValidationError)
with self.assertRaises(GCPReportDownloaderError):
GCPReportDownloader(FAKE.name(), {'bucket': FAKE.slug()})
