def test_get_dataset_name(self): """Test _get_dataset_name helper.""" project_id = FAKE.slug() dataset_name = FAKE.slug() datasets = [f"{project_id}:{dataset_name}", dataset_name] for dataset in datasets: billing_source = {"table_id": FAKE.slug(), "dataset": dataset} credentials = {"project_id": project_id} with patch( "masu.external.downloader.gcp.gcp_report_downloader.GCPProvider" ), patch( "masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag", return_value=self.etag, ): with patch( "masu.external.downloader.gcp.gcp_report_downloader.GCPProvider" ): downloader = GCPReportDownloader( customer_name=FAKE.name(), data_source=billing_source, provider_uuid=uuid4(), credentials=credentials, ) self.assertEqual(downloader._get_dataset_name(), dataset_name)
def test_init_unreachable_bucket_raises_error(self, mock_provider): """Assert GCPReportDownloader raises error when bucket is not reachable.""" mock_provider.return_value.cost_usage_source_is_reachable.side_effect = ValidationError with self.assertRaises(GCPReportDownloaderError): GCPReportDownloader( Mock(request=Mock(id=str(FAKE.uuid4()), return_value={})), FAKE.name(), {"bucket": FAKE.slug()})
def test_download_with_unreachable_source(self, gcp_provider): """Assert errors correctly when source is unreachable.""" gcp_provider.return_value.cost_usage_source_is_reachable.side_effect = ValidationError billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()} credentials = {"project_id": FAKE.slug()} with self.assertRaises(GCPReportDownloaderError): GCPReportDownloader(FAKE.name(), billing_source, credentials=credentials)
def test_generate_etag(self, gcp_provider): """Test BigQuery client is handled correctly in generate etag method.""" billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()} credentials = {"project_id": FAKE.slug()} with patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery") as bigquery: bigquery.Client.return_value.get_table.return_value.modified.return_value = self.today downloader = GCPReportDownloader( customer_name=FAKE.name(), data_source=billing_source, provider_uuid=uuid4(), credentials=credentials ) self.assertIsNotNone(downloader.etag)
def _set_downloader(self): """ Create the report downloader object. Downloader is specific to the provider's cloud service. Args: None Returns: (Object) : Some object that is a child of CURAccountsInterface """ if self.provider_type == AMAZON_WEB_SERVICES: return AWSReportDownloader(customer_name=self.customer_name, auth_credential=self.credential, bucket=self.cur_source, report_name=self.report_name, provider_id=self.provider_id) if self.provider_type == AWS_LOCAL_SERVICE_PROVIDER: return AWSLocalReportDownloader(customer_name=self.customer_name, auth_credential=self.credential, bucket=self.cur_source, report_name=self.report_name, provider_id=self.provider_id) if self.provider_type == AZURE: return AzureReportDownloader(customer_name=self.customer_name, auth_credential=self.credential, billing_source=self.cur_source, report_name=self.report_name, provider_id=self.provider_id) if self.provider_type == AZURE_LOCAL_SERVICE_PROVIDER: return AzureLocalReportDownloader(customer_name=self.customer_name, auth_credential=self.credential, billing_source=self.cur_source, report_name=self.report_name, provider_id=self.provider_id) if self.provider_type == OPENSHIFT_CONTAINER_PLATFORM: return OCPReportDownloader(customer_name=self.customer_name, auth_credential=self.credential, bucket=self.cur_source, report_name=self.report_name, provider_id=self.provider_id) if self.provider_type == GCP: return GCPReportDownloader(customer_name=self.customer_name, auth_credential=self.credential, billing_source=self.cur_source, report_name=self.report_name, provider_id=self.provider_id) return None
def test_generate_etag_big_query_client_error(self, gcp_provider): """Test BigQuery client is handled correctly in generate etag method.""" billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()} credentials = {"project_id": FAKE.slug()} err_msg = "GCP Error" with patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery") as bigquery: bigquery.Client.side_effect = GoogleCloudError(err_msg) with self.assertRaisesRegexp(GCPReportDownloaderError, err_msg): GCPReportDownloader( customer_name=FAKE.name(), data_source=billing_source, provider_uuid=uuid4(), credentials=credentials, )
def test_init_reachable_bucket_is_okay(self): """Assert GCPReportDownloader initializes with expected values.""" customer_name = FAKE.name() bucket_name = FAKE.slug() billing_source = {"bucket": bucket_name} with patch( "masu.external.downloader.gcp.gcp_report_downloader.GCPProvider" ), patch("masu.external.downloader.gcp.gcp_report_downloader.storage" ) as mock_storage: downloader = GCPReportDownloader(customer_name, billing_source) mock_storage.Client.return_value.lookup_bucket.assert_called_with( bucket_name) self.assertEqual(downloader.customer_name, customer_name.replace(" ", "_")) self.assertEqual(downloader.bucket_name, bucket_name)
def test_init_reachable_bucket_is_okay(self): """Assert GCPReportDownloader initializes with expected values.""" customer_name = FAKE.name() bucket_name = FAKE.slug() billing_source = {'bucket': bucket_name} with patch( 'masu.external.downloader.gcp.gcp_report_downloader.GCPProvider' ) as mock_provider, patch( 'masu.external.downloader.gcp.gcp_report_downloader.storage' ) as mock_storage: downloader = GCPReportDownloader(Mock(request=Mock(id=str(FAKE.uuid4()), return_value={})), customer_name, billing_source) mock_storage.Client.return_value.lookup_bucket.assert_called_with( bucket_name ) self.assertEqual(downloader.customer_name, customer_name.replace(' ', '_')) self.assertEqual(downloader.bucket_name, bucket_name)
def create_gcp_downloader_with_mock_gcp_storage( self, customer_name=None, bucket_name=None, provider_uuid=None, report_prefix=None ): """ Create a GCPReportDownloader instance that skips the initial GCP client/bucket check. This also results in Mock objects being set to instance variables that can be patched inside other test functions. Args: customer_name (str): optional customer name; will be randomly generated if None bucket_name (str): optional bucket name; will be randomly generated if None provider_uuid (uuid): optional provider UUID; will be randomly generated if None report_prefix (str): optional report prefix Returns: GCPReportDownloader instance with faked argument data and Mocks in self._storage_client and self._bucket_info. """ if not customer_name: customer_name = FAKE.name() if not bucket_name: bucket_name = FAKE.slug() billing_source = {'bucket': bucket_name} if report_prefix: billing_source['report_prefix'] = report_prefix if not provider_uuid: provider_uuid = uuid4() with patch( 'masu.external.downloader.gcp.gcp_report_downloader.GCPProvider' ) as mock_provider, patch( 'masu.external.downloader.gcp.gcp_report_downloader.storage' ) as mock_storage: # mock_storage_client = mock_storage.Client.return_value # mock_storage_client.lookup_bucket.return_value = {} downloader = GCPReportDownloader( task=Mock(request=Mock(id=str(FAKE.uuid4()), return_value={})), customer_name=customer_name, billing_source=billing_source, provider_uuid=provider_uuid, ) return downloader
def create_gcp_downloader_with_mocked_values( self, customer_name=FAKE.name(), dataset=FAKE.slug(), provider_uuid=uuid4(), project_id=FAKE.slug(), table_id=FAKE.slug(), ): """ Create a GCPReportDownloader that skips the initial GCP bigquery check creates etag. This also results in Mock objects being set to instance variables that can be patched inside other test functions. Args: customer_name (str): optional customer name; will be randomly generated if None bucket_name (str): optional bucket name; will be randomly generated if None provider_uuid (uuid): optional provider UUID; will be randomly generated if None Returns: GCPReportDownloader instance with faked argument data and Mocks in self.etag. """ billing_source = {"table_id": table_id, "dataset": dataset} credentials = {"project_id": project_id} with patch( "masu.external.downloader.gcp.gcp_report_downloader.GCPProvider" ), patch( "masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag", return_value=self.etag, ): downloader = GCPReportDownloader( customer_name=customer_name, data_source=billing_source, provider_uuid=provider_uuid, credentials=credentials, ) return downloader
def _set_downloader(self): """ Create the report downloader object. Downloader is specific to the provider's cloud service. Args: None Returns: (Object) : Some object that is a child of CURAccountsInterface """ if self.provider_type == Provider.PROVIDER_AWS: return AWSReportDownloader( customer_name=self.customer_name, auth_credential=self.credential, bucket=self.cur_source, report_name=self.report_name, provider_uuid=self.provider_uuid, request_id=self.request_id, account=self.account, ) if self.provider_type == Provider.PROVIDER_AWS_LOCAL: return AWSLocalReportDownloader( customer_name=self.customer_name, auth_credential=self.credential, bucket=self.cur_source, report_name=self.report_name, provider_uuid=self.provider_uuid, request_id=self.request_id, account=self.account, ) if self.provider_type == Provider.PROVIDER_AZURE: return AzureReportDownloader( customer_name=self.customer_name, auth_credential=self.credential, billing_source=self.cur_source, report_name=self.report_name, provider_uuid=self.provider_uuid, request_id=self.request_id, account=self.account, ) if self.provider_type == Provider.PROVIDER_AZURE_LOCAL: return AzureLocalReportDownloader( customer_name=self.customer_name, auth_credential=self.credential, billing_source=self.cur_source, report_name=self.report_name, provider_uuid=self.provider_uuid, request_id=self.request_id, account=self.account, ) if self.provider_type == Provider.PROVIDER_OCP: return OCPReportDownloader( customer_name=self.customer_name, auth_credential=self.credential, bucket=self.cur_source, report_name=self.report_name, provider_uuid=self.provider_uuid, request_id=self.request_id, account=self.account, ) if self.provider_type == Provider.PROVIDER_GCP: return GCPReportDownloader( customer_name=self.customer_name, auth_credential=self.credential, billing_source=self.cur_source, report_name=self.report_name, provider_uuid=self.provider_uuid, request_id=self.request_id, account=self.account, ) return None
def test_init_unreachable_bucket_raises_error(self, mock_provider): """Assert GCPReportDownloader raises error when bucket is not reachable.""" mock_provider.return_value.cost_usage_source_is_reachable.side_effect = ( ValidationError) with self.assertRaises(GCPReportDownloaderError): GCPReportDownloader(FAKE.name(), {'bucket': FAKE.slug()})