def setUp(self, mock_service):
"""Set up each test."""
mock_service.return_value = MockAzureService()
super().setUp()
self.customer_name = "Azure Customer"
self.auth_credential = self.azure_credentials
self.billing_source = self.azure_data_source
self.downloader = AzureReportDownloader(
customer_name=self.customer_name,
auth_credential=self.auth_credential,
billing_source=self.billing_source,
provider_id=self.azure_provider_id)
self.mock_data = MockAzureService()
def setUp(self, mock_service):
"""Set up each test."""
mock_service.return_value = MockAzureService()
super().setUp()
self.customer_name = "Azure Customer"
self.auth_credential = self.azure_credentials
self.billing_source = self.azure_data_source
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.downloader = AzureReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
auth_credential=self.auth_credential,
billing_source=self.billing_source,
provider_uuid=self.azure_provider_uuid)
self.mock_data = MockAzureService()
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == AMAZON_WEB_SERVICES:
return AWSReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AWS_LOCAL_SERVICE_PROVIDER:
return AWSLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE:
return AzureReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE_LOCAL_SERVICE_PROVIDER:
return AzureLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == OPENSHIFT_CONTAINER_PLATFORM:
return OCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == GCP:
return GCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
return None
def test_init_with_demo_account(self, mock_download_cost_method, _):
"""Test init with the demo account."""
account_id = "123456"
report_name = self.fake.word()
client_id = self.azure_credentials.get("client_id")
demo_accounts = {
account_id: {
client_id: {
"report_name": report_name,
"report_prefix": self.fake.word(),
"container_name": self.fake.word(),
}
}
}
with self.settings(DEMO_ACCOUNTS=demo_accounts):
AzureReportDownloader(
customer_name=f"acct{account_id}",
credentials=self.azure_credentials,
data_source=self.azure_data_source,
provider_uuid=self.azure_provider_uuid,
)
mock_download_cost_method._azure_client.download_cost_export.assert_not_called()
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == Provider.PROVIDER_AWS:
return AWSReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AWS_LOCAL:
return AWSLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE:
return AzureReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE_LOCAL:
return AzureLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_OCP:
return OCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_GCP:
return GCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
return None
class AzureReportDownloaderTest(MasuTestCase):
"""Test Cases for the AzureReportDownloader object."""
fake = Faker()
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureService")
def setUp(self, mock_service):
"""Set up each test."""
mock_service.return_value = MockAzureService()
super().setUp()
self.customer_name = "Azure Customer"
self.azure_credentials = self.azure_provider.authentication.credentials
self.azure_data_source = self.azure_provider.billing_source.data_source
self.downloader = AzureReportDownloader(
customer_name=self.customer_name,
credentials=self.azure_credentials,
data_source=self.azure_data_source,
provider_uuid=self.azure_provider_uuid,
)
self.mock_data = MockAzureService()
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(DATA_DIR, ignore_errors=True)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureService",
return_value=MockAzureService())
def test_get_azure_client(self, _):
"""Test to verify Azure downloader is initialized."""
client = self.downloader._get_azure_client(self.azure_credentials,
self.azure_data_source)
self.assertIsNotNone(client)
def test_get_report_path(self):
"""Test that report path is built correctly."""
self.assertEqual(self.downloader.directory, self.mock_data.directory)
self.assertEqual(self.downloader.export_name,
self.mock_data.export_name)
self.assertEqual(
self.downloader._get_report_path(self.mock_data.test_date),
self.mock_data.report_path)
def test_get_local_file_for_report(self):
"""Test to get the local file path for a report."""
expected_local_file = self.mock_data.export_file
local_file = self.downloader.get_local_file_for_report(
self.mock_data.export_key)
self.assertEqual(expected_local_file, local_file)
def test_get_manifest(self):
"""Test that Azure manifest is created."""
expected_start, expected_end = self.mock_data.month_range.split("-")
manifest, _ = self.downloader._get_manifest(self.mock_data.test_date)
self.assertEqual(manifest.get("assemblyId"),
self.mock_data.export_uuid)
self.assertEqual(manifest.get("reportKeys"),
[self.mock_data.export_file])
self.assertEqual(manifest.get("Compression"), "PLAIN")
self.assertEqual(
manifest.get("billingPeriod").get("start"), expected_start)
self.assertEqual(
manifest.get("billingPeriod").get("end"), expected_end)
def test_get_manifest_unexpected_report_name(self):
"""Test that error is thrown when getting manifest with an unexpected report name."""
with self.assertRaises(AzureReportDownloaderError):
self.downloader._get_manifest(self.mock_data.bad_test_date)
def test_download_file(self):
"""Test that Azure report report is downloaded."""
expected_full_path = "{}/{}/azure/{}/{}".format(
Config.TMP_DIR, self.customer_name.replace(" ", "_"),
self.mock_data.container, self.mock_data.export_file)
full_file_path, etag, _, __ = self.downloader.download_file(
self.mock_data.export_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertEqual(etag, self.mock_data.export_etag)
def test_download_missing_file(self):
"""Test that Azure report is not downloaded for incorrect key."""
key = "badkey"
with self.assertRaises(AzureReportDownloaderError):
self.downloader.download_file(key)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureReportDownloader"
)
def test_download_file_matching_etag(self, mock_download_cost_method):
"""Test that Azure report report is not downloaded with matching etag."""
expected_full_path = "{}/{}/azure/{}/{}".format(
Config.TMP_DIR, self.customer_name.replace(" ", "_"),
self.mock_data.container, self.mock_data.export_file)
full_file_path, etag, _, __ = self.downloader.download_file(
self.mock_data.export_key, self.mock_data.export_etag)
self.assertEqual(full_file_path, expected_full_path)
self.assertEqual(etag, self.mock_data.export_etag)
mock_download_cost_method._azure_client.download_cost_export.assert_not_called(
)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureReportDownloader"
)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureService",
return_value=MockAzureService())
def test_init_with_demo_account(self, mock_download_cost_method, _):
"""Test init with the demo account."""
account_id = "123456"
report_name = self.fake.word()
client_id = self.azure_credentials.get("client_id")
demo_accounts = {
account_id: {
client_id: {
"report_name": report_name,
"report_prefix": self.fake.word(),
"container_name": self.fake.word(),
}
}
}
with self.settings(DEMO_ACCOUNTS=demo_accounts):
AzureReportDownloader(
customer_name=f"acct{account_id}",
credentials=self.azure_credentials,
data_source=self.azure_data_source,
provider_uuid=self.azure_provider_uuid,
)
mock_download_cost_method._azure_client.download_cost_export.assert_not_called(
)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureReportDownloader._get_manifest"
)
def test_get_manifest_context_for_date(self, mock_manifest):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
start_str = current_month.strftime(
self.downloader.manifest_date_format)
assembly_id = "1234"
compression = UNCOMPRESSED
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {
"start": start_str
},
},
DateAccessor().today(),
)
result = self.downloader.get_manifest_context_for_date(current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
class AzureReportDownloaderTest(MasuTestCase):
"""Test Cases for the AzureReportDownloader object."""
fake = Faker()
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureService")
def setUp(self, mock_service):
"""Set up each test."""
mock_service.return_value = MockAzureService()
super().setUp()
self.customer_name = "Azure Customer"
self.azure_credentials = self.azure_provider.authentication.credentials
self.azure_data_source = self.azure_provider.billing_source.data_source
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.downloader = AzureReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
auth_credential=self.azure_credentials,
billing_source=self.azure_data_source,
provider_uuid=self.azure_provider_uuid,
)
self.mock_data = MockAzureService()
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(DATA_DIR, ignore_errors=True)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureService",
return_value=MockAzureService())
def test_get_azure_client(self, _):
"""Test to verify Azure downloader is initialized."""
client = self.downloader._get_azure_client(self.azure_credentials,
self.azure_data_source)
self.assertIsNotNone(client)
def test_get_report_path(self):
"""Test that report path is built correctly."""
self.assertEqual(self.downloader.directory, self.mock_data.directory)
self.assertEqual(self.downloader.export_name,
self.mock_data.export_name)
self.assertEqual(
self.downloader._get_report_path(self.mock_data.test_date),
self.mock_data.report_path)
def test_get_local_file_for_report(self):
"""Test to get the local file path for a report."""
expected_local_file = self.mock_data.export_file
local_file = self.downloader.get_local_file_for_report(
self.mock_data.export_key)
self.assertEqual(expected_local_file, local_file)
def test_get_manifest(self):
"""Test that Azure manifest is created."""
expected_start, expected_end = self.mock_data.month_range.split("-")
manifest = self.downloader._get_manifest(self.mock_data.test_date)
self.assertEqual(manifest.get("assemblyId"),
self.mock_data.export_uuid)
self.assertEqual(manifest.get("reportKeys"),
[self.mock_data.export_file])
self.assertEqual(manifest.get("Compression"), "PLAIN")
self.assertEqual(
manifest.get("billingPeriod").get("start"), expected_start)
self.assertEqual(
manifest.get("billingPeriod").get("end"), expected_end)
def test_get_manifest_unexpected_report_name(self):
"""Test that error is thrown when getting manifest with an unexpected report name."""
with self.assertRaises(AzureReportDownloaderError):
self.downloader._get_manifest(self.mock_data.bad_test_date)
def test_get_report_context_for_date_should_download(self):
"""Test that report context is retrieved for date."""
with patch.object(ReportDownloaderBase,
"check_if_manifest_should_be_downloaded",
return_value=True):
manifest = self.downloader.get_report_context_for_date(
self.mock_data.test_date)
self.assertEqual(manifest.get("assembly_id"),
self.mock_data.export_uuid)
self.assertEqual(manifest.get("compression"), "PLAIN")
self.assertEqual(manifest.get("files"), [self.mock_data.export_file])
self.assertIsNotNone(manifest.get("manifest_id"))
def test_get_report_context_for_date_should_not_download(self):
"""Test that report context is not retrieved when download check fails."""
with patch.object(ReportDownloaderBase,
"check_if_manifest_should_be_downloaded",
return_value=False):
manifest = self.downloader.get_report_context_for_date(
self.mock_data.test_date)
self.assertEqual(manifest, {})
def test_get_report_context_for_incorrect_date(self):
"""Test that report context is not retrieved with an unexpected date."""
test_date = datetime(2019, 9, 15)
with patch.object(ReportDownloaderBase,
"check_if_manifest_should_be_downloaded",
return_value=False):
manifest = self.downloader.get_report_context_for_date(test_date)
self.assertEqual(manifest, {})
def test_download_file(self):
"""Test that Azure report report is downloaded."""
expected_full_path = "{}/{}/azure/{}/{}".format(
Config.TMP_DIR, self.customer_name.replace(" ", "_"),
self.mock_data.container, self.mock_data.export_file)
full_file_path, etag = self.downloader.download_file(
self.mock_data.export_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertEqual(etag, self.mock_data.export_etag)
def test_download_missing_file(self):
"""Test that Azure report is not downloaded for incorrect key."""
key = "badkey"
with self.assertRaises(AzureReportDownloaderError):
self.downloader.download_file(key)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureReportDownloader"
)
def test_download_file_matching_etag(self, mock_download_cost_method):
"""Test that Azure report report is not downloaded with matching etag."""
expected_full_path = "{}/{}/azure/{}/{}".format(
Config.TMP_DIR, self.customer_name.replace(" ", "_"),
self.mock_data.container, self.mock_data.export_file)
full_file_path, etag = self.downloader.download_file(
self.mock_data.export_key, self.mock_data.export_etag)
self.assertEqual(full_file_path, expected_full_path)
self.assertEqual(etag, self.mock_data.export_etag)
mock_download_cost_method._azure_client.download_cost_export.assert_not_called(
)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureReportDownloader"
)
@patch(
"masu.external.downloader.azure.azure_report_downloader.AzureService",
return_value=MockAzureService())
def test_init_with_demo_account(self, mock_download_cost_method, _):
"""Test init with the demo account."""
account_id = "123456"
report_name = self.fake.word()
client_id = self.azure_credentials.get("client_id")
demo_accounts = {
account_id: {
client_id: {
"report_name": report_name,
"report_prefix": self.fake.word(),
"container_name": self.fake.word(),
}
}
}
with self.settings(DEMO_ACCOUNTS=demo_accounts):
AzureReportDownloader(
task=self.mock_task,
customer_name=f"acct{account_id}",
auth_credential=self.azure_credentials,
billing_source=self.azure_data_source,
provider_uuid=self.azure_provider_uuid,
)
mock_download_cost_method._azure_client.download_cost_export.assert_not_called(
)
class AzureReportDownloaderTest(MasuTestCase):
"""Test Cases for the AzureReportDownloader object."""
@patch(
'masu.external.downloader.azure.azure_report_downloader.AzureService')
def setUp(self, mock_service):
"""Set up each test."""
mock_service.return_value = MockAzureService()
super().setUp()
self.customer_name = "Azure Customer"
self.auth_credential = self.azure_credentials
self.billing_source = self.azure_data_source
self.downloader = AzureReportDownloader(
customer_name=self.customer_name,
auth_credential=self.auth_credential,
billing_source=self.billing_source,
provider_id=self.azure_provider_id)
self.mock_data = MockAzureService()
def tearDown(self):
super().tearDown()
shutil.rmtree(DATA_DIR, ignore_errors=True)
@patch(
'masu.external.downloader.azure.azure_report_downloader.AzureService',
return_value=MockAzureService())
def test_get_azure_client(self, mock_service):
"""Test to verify Azure downloader is initialized."""
client = self.downloader._get_azure_client(self.azure_credentials,
self.azure_data_source)
self.assertIsNotNone(client)
def test_get_report_path(self):
"""Test that report path is built correctly."""
self.assertEqual(self.downloader.directory, self.mock_data.directory)
self.assertEqual(self.downloader.export_name,
self.mock_data.export_name)
self.assertEqual(
self.downloader._get_report_path(self.mock_data.test_date),
self.mock_data.report_path)
def test_get_local_file_for_report(self):
"""Test to get the local file path for a report."""
expected_local_file = self.mock_data.export_file
local_file = self.downloader.get_local_file_for_report(
self.mock_data.export_key)
self.assertEqual(expected_local_file, local_file)
def test_get_manifest(self):
"""Test that Azure manifest is created."""
expected_start, expected_end = self.mock_data.month_range.split('-')
manifest = self.downloader._get_manifest(self.mock_data.test_date)
self.assertEqual(manifest.get('assemblyId'),
self.mock_data.export_uuid)
self.assertEqual(manifest.get('reportKeys'),
[self.mock_data.export_file])
self.assertEqual(manifest.get('Compression'), 'PLAIN')
self.assertEqual(
manifest.get('billingPeriod').get('start'), expected_start)
self.assertEqual(
manifest.get('billingPeriod').get('end'), expected_end)
expected_manifest_path = '{}/{}'.format(
self.downloader._get_exports_data_directory(), 'Manifest.json')
self.assertTrue(Path(expected_manifest_path).exists())
def test_get_manifest_unexpected_report_name(self):
"""Test that error is thrown when getting manifest with an unexpected report name."""
with self.assertRaises(AzureReportDownloaderError):
self.downloader._get_manifest(self.mock_data.bad_test_date)
def test_get_report_context_for_date_should_download(self):
"""Test that report context is retrieved for date."""
with patch.object(ReportDownloaderBase,
'check_if_manifest_should_be_downloaded',
return_value=True):
manifest = self.downloader.get_report_context_for_date(
self.mock_data.test_date)
self.assertEqual(manifest.get('assembly_id'),
self.mock_data.export_uuid)
self.assertEqual(manifest.get('compression'), 'PLAIN')
self.assertEqual(manifest.get('files'), [self.mock_data.export_file])
self.assertIsNotNone(manifest.get('manifest_id'))
def test_get_report_context_for_date_should_not_download(self):
"""Test that report context is not retrieved when download check fails."""
with patch.object(ReportDownloaderBase,
'check_if_manifest_should_be_downloaded',
return_value=False):
manifest = self.downloader.get_report_context_for_date(
self.mock_data.test_date)
self.assertEqual(manifest, {})
def test_get_report_context_for_incorrect_date(self):
"""Test that report context is not retrieved with an unexpected date."""
test_date = datetime(2019, 9, 15)
with patch.object(ReportDownloaderBase,
'check_if_manifest_should_be_downloaded',
return_value=False):
manifest = self.downloader.get_report_context_for_date(test_date)
self.assertEqual(manifest, {})
def test_download_file(self):
"""Test that Azure report report is downloaded."""
expected_full_path = '{}/{}/azure/{}/{}'.format(
Config.TMP_DIR, self.customer_name.replace(' ', '_'),
self.mock_data.container, self.mock_data.export_file)
full_file_path, etag = self.downloader.download_file(
self.mock_data.export_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertEqual(etag, self.mock_data.export_etag)
def test_download_missing_file(self):
"""Test that Azure report is not downloaded for incorrect key."""
key = 'badkey'
with self.assertRaises(AzureReportDownloaderError):
self.downloader.download_file(key)
@patch(
'masu.external.downloader.azure.azure_report_downloader.AzureReportDownloader'
)
def test_download_file_matching_etag(self, mock_download_cost_method):
"""Test that Azure report report is not downloaded with matching etag."""
expected_full_path = '{}/{}/azure/{}/{}'.format(
Config.TMP_DIR, self.customer_name.replace(' ', '_'),
self.mock_data.container, self.mock_data.export_file)
full_file_path, etag = self.downloader.download_file(
self.mock_data.export_key, self.mock_data.export_etag)
self.assertEqual(full_file_path, expected_full_path)
self.assertEqual(etag, self.mock_data.export_etag)
mock_download_cost_method._azure_client.download_cost_export.assert_not_called(
)