def test_download_cost_report_exception(self, mock_factory):
"""Test that function handles a raised exception."""
key = FAKE.word()
mock_blob = Mock(last_modified=Mock(date=Mock(
return_value=datetime.now())))
name_attr = PropertyMock(return_value=key)
type(
mock_blob).name = name_attr # kludge to set name attribute on Mock
mock_factory.return_value = Mock(
spec=AzureClientFactory,
cloud_storage_account=Mock(return_value=Mock(
spec=BlobServiceClient,
get_blob_client=Mock(side_effect=AdalError("test error")),
get_container_client=Mock(
return_value=Mock(spec=ContainerClient,
list_blobs=Mock(
return_value=[mock_blob]))),
)),
)
with self.assertRaises(AzureServiceError):
service = AzureService(self.tenant_id, self.client_id,
self.client_secret,
self.resource_group_name,
self.storage_account_name)
service.download_cost_export(key=key, container_name=FAKE.word())
def setUp(self, mock_factory):
"""Set up each test."""
super().setUp()
self.subscription_id = FAKE.uuid4()
self.tenant_id = FAKE.uuid4()
self.client_id = FAKE.uuid4()
self.client_secret = FAKE.word()
self.resource_group_name = FAKE.word()
self.storage_account_name = FAKE.word()
self.container_name = FAKE.word()
self.current_date_time = datetime.today()
self.export_directory = FAKE.word()
mock_factory.return_value = MockAzureClientFactory(
self.subscription_id,
self.container_name,
self.current_date_time,
self.export_directory,
self.resource_group_name,
self.storage_account_name,
)
self.client = AzureService(
subscription_id=self.subscription_id,
tenant_id=self.tenant_id,
client_id=self.client_id,
client_secret=self.client_secret,
resource_group_name=self.resource_group_name,
storage_account_name=self.storage_account_name,
)
def cost_usage_source_is_reachable(self, credential_name, storage_resource_name):
"""
Verify that the cost usage report source is reachable by Koku.
Implemented by provider specific class. An account validation and
connectivity check is to be done.
Args:
credential (dict): Azure credentials dict
example: {'subscription_id': 'f695f74f-36a4-4112-9fe6-74415fac75a2',
'tenant_id': '319d4d72-7ddc-45d0-9d63-a2db0a36e048',
'client_id': 'ce26bd50-2e5a-4eb7-9504-a05a79568e25',
'client_secret': 'abc123' }
source_name (dict): Identifier of the cost usage report source
example: { 'resource_group': 'My Resource Group 1',
'storage_account': 'My Storage Account 2'
Returns:
None
Raises:
ValidationError: Error string
"""
key = "billing_source.data_source"
azure_service = None
if not (isinstance(credential_name, dict) and isinstance(storage_resource_name, dict)):
message = f"Resource group and/or Storage account must be a dict"
raise ValidationError(error_obj(key, message))
resource_group = storage_resource_name.get("resource_group")
storage_account = storage_resource_name.get("storage_account")
if not (resource_group and storage_account):
message = "resource_group or storage_account is undefined."
raise ValidationError(error_obj(key, message))
try:
azure_service = AzureService(
**credential_name, resource_group_name=resource_group, storage_account_name=storage_account
)
azure_client = AzureClientFactory(**credential_name)
storage_accounts = azure_client.storage_client.storage_accounts
storage_account = storage_accounts.get_properties(resource_group, storage_account)
if azure_service and not azure_service.describe_cost_management_exports():
message = "Cost management export was not found."
raise ValidationError(error_obj(key, message))
except AzureCostReportNotFound as costreport_err:
raise ValidationError(error_obj(key, str(costreport_err)))
except (AdalError, AzureException, AzureServiceError, ClientException, TypeError) as exc:
raise ValidationError(error_obj(key, str(exc)))
return True
def test_describe_cost_management_exports_wrong_account(
self, mock_factory):
"""Test that cost management exports are not returned from incorrect account."""
mock_factory.return_value = MockAzureClientFactory(
self.subscription_id, self.container_name, self.current_date_time,
self.export_directory, self.resource_group_name,
self.storage_account_name)
client = AzureService(subscription_id=self.subscription_id,
tenant_id=self.tenant_id,
client_id=self.client_id,
client_secret=self.client_secret,
resource_group_name=self.resource_group_name,
storage_account_name='wrongaccount')
exports = client.describe_cost_management_exports()
self.assertEquals(exports, [])
def test_get_latest_cost_export_for_path_exception(self, mock_factory):
"""Test that function handles a raised exception."""
mock_factory.return_value = Mock(
spec=AzureClientFactory,
cloud_storage_account=Mock(return_value=Mock(
spec=BlobServiceClient,
get_container_client=Mock(return_value=Mock(
spec=ContainerClient,
list_blobs=Mock(side_effect=AdalError("test error")))),
)),
)
with self.assertRaises(AzureServiceError):
service = AzureService(self.tenant_id, self.client_id,
self.client_secret,
self.resource_group_name,
self.storage_account_name)
service.get_latest_cost_export_for_path(report_path=FAKE.word(),
container_name=FAKE.word())
def test_init_no_subscription_id(self, mock_factory):
"""Test that exception is raized with no subscription id provided."""
class MockAzureFactory:
subscription_id = None
factory = MockAzureFactory()
mock_factory.return_value = factory
with self.assertRaises(AzureServiceError):
AzureService(self.tenant_id, self.client_id, self.client_secret,
self.resource_group_name, self.storage_account_name)
def _get_azure_client(credentials, billing_source):
subscription_id = credentials.get('subscription_id')
tenant_id = credentials.get('tenant_id')
client_id = credentials.get('client_id')
client_secret = credentials.get('client_secret')
resource_group_name = billing_source.get('resource_group')
storage_account_name = billing_source.get('storage_account')
service = AzureService(subscription_id, tenant_id, client_id, client_secret,
resource_group_name, storage_account_name)
return service
def _get_azure_client(credentials, data_source):
subscription_id = credentials.get("subscription_id")
tenant_id = credentials.get("tenant_id")
client_id = credentials.get("client_id")
client_secret = credentials.get("client_secret")
resource_group_name = data_source.get("resource_group")
storage_account_name = data_source.get("storage_account")
service = AzureService(tenant_id, client_id, client_secret,
resource_group_name, storage_account_name,
subscription_id)
return service
class AzureServiceTest(MasuTestCase):
"""Test Cases for the AzureService object."""
@patch("masu.external.downloader.azure.azure_service.AzureClientFactory")
def setUp(self, mock_factory):
"""Set up each test."""
super().setUp()
self.subscription_id = FAKE.uuid4()
self.tenant_id = FAKE.uuid4()
self.client_id = FAKE.uuid4()
self.client_secret = FAKE.word()
self.resource_group_name = FAKE.word()
self.storage_account_name = FAKE.word()
self.container_name = FAKE.word()
self.current_date_time = datetime.today()
self.export_directory = FAKE.word()
mock_factory.return_value = MockAzureClientFactory(
self.subscription_id,
self.container_name,
self.current_date_time,
self.export_directory,
self.resource_group_name,
self.storage_account_name,
)
self.client = AzureService(
subscription_id=self.subscription_id,
tenant_id=self.tenant_id,
client_id=self.client_id,
client_secret=self.client_secret,
resource_group_name=self.resource_group_name,
storage_account_name=self.storage_account_name,
)
def test_initializer(self):
"""Test the AzureService initializer."""
self.assertIsNotNone(self.client._factory)
self.assertIsNotNone(self.client._cloud_storage_account)
def test_get_cost_export_for_key(self):
"""Test that a cost export is retrieved by a key."""
today = self.current_date_time
yesterday = today - relativedelta(days=1)
test_matrix = [
{
"key":
"{}_{}_day_{}".format(self.container_name, "blob", today.day),
"expected_date":
today.date()
},
{
"key":
"{}_{}_day_{}".format(self.container_name, "blob",
yesterday.day),
"expected_date":
yesterday.date(),
},
]
for test in test_matrix:
key = test.get("key")
expected_modified_date = test.get("expected_date")
cost_export = self.client.get_cost_export_for_key(
key, self.container_name)
self.assertIsNotNone(cost_export)
self.assertEquals(cost_export.name, key)
self.assertEquals(cost_export.properties.last_modified.date(),
expected_modified_date)
def test_get_cost_export_for_missing_key(self):
"""Test that a cost export is not retrieved by an incorrect key."""
key = "{}_{}_wrong".format(self.container_name, "blob")
with self.assertRaises(AzureCostReportNotFound):
self.client.get_cost_export_for_key(key, self.container_name)
def test_get_latest_cost_export_for_path(self):
"""Test that the latest cost export is returned for a given path."""
report_path = "{}_{}".format(self.container_name, "blob")
cost_export = self.client.get_latest_cost_export_for_path(
report_path, self.container_name)
self.assertEquals(cost_export.properties.last_modified.date(),
self.current_date_time.date())
def test_get_latest_cost_export_for_path_missing(self):
"""Test that the no cost export is returned for a missing path."""
report_path = FAKE.word()
with self.assertRaises(AzureCostReportNotFound):
self.client.get_latest_cost_export_for_path(
report_path, self.container_name)
def test_describe_cost_management_exports(self):
"""Test that cost management exports are returned for the account."""
exports = self.client.describe_cost_management_exports()
self.assertEquals(len(exports), 2)
for export in exports:
self.assertEquals(export.get("container"), self.container_name)
self.assertEquals(export.get("directory"), self.export_directory)
self.assertIn("{}_{}".format(self.container_name, "blob"),
export.get("name"))
@patch("masu.external.downloader.azure.azure_service.AzureClientFactory")
def test_describe_cost_management_exports_wrong_account(
self, mock_factory):
"""Test that cost management exports are not returned from incorrect account."""
mock_factory.return_value = MockAzureClientFactory(
self.subscription_id,
self.container_name,
self.current_date_time,
self.export_directory,
self.resource_group_name,
self.storage_account_name,
)
client = AzureService(
subscription_id=self.subscription_id,
tenant_id=self.tenant_id,
client_id=self.client_id,
client_secret=self.client_secret,
resource_group_name=self.resource_group_name,
storage_account_name="wrongaccount",
)
exports = client.describe_cost_management_exports()
self.assertEquals(exports, [])
def test_download_cost_export(self):
"""Test that cost management exports are downloaded."""
key = "{}_{}_day_{}".format(self.container_name, "blob",
self.current_date_time.day)
file_path = self.client.download_cost_export(key, self.container_name)
self.assertTrue(file_path.endswith(".csv"))
def get_mock_client(self, blob_list=[], cost_exports=[]):
"""Generate an AzureService instance with mocked AzureClientFactory.
Args:
blob_list (list<Mock>) A list of Mock objects.
The blob_list Mock objects must have these attributes:
- name
cost_exports (list<Mock>) A list of Mock objects.
The cost_exports Mock objects must have these
attributes:
- name
- delivery_info.destination.container
- delivery_info.destination.root_folder_path
- delivery_info.destination.resource_id
Returns:
(AzureService) An instance of AzureService with mocked AzureClientFactory
"""
fake_data = FAKE.binary(length=1024 * 64)
client = None
with patch(
"masu.external.downloader.azure.azure_service.AzureClientFactory",
spec=AzureClientFactory) as mock_factory:
mock_factory.return_value = Mock( # AzureClientFactory()
spec=AzureClientFactory,
cloud_storage_account=Mock(
return_value=Mock( # .cloud_storage_account()
spec=BlobServiceClient,
get_blob_client=Mock(
return_value=Mock( # .get_blob_client()
spec=BlobClient,
# .download_blob().readall()
download_blob=Mock(return_value=Mock(
readall=Mock(return_value=fake_data))),
)),
get_container_client=Mock(
# .get_container_client().list_blobs()
return_value=Mock(spec=ContainerClient,
list_blobs=Mock(
return_value=blob_list))),
)),
# .cost_management_client.exports.list().value
cost_management_client=Mock(exports=Mock(list=Mock(
return_value=Mock(value=cost_exports)))),
# .subscription_id
subscription_id=self.subscription_id,
)
client = AzureService(
self.tenant_id,
self.client_id,
self.client_secret,
self.resource_group_name,
self.storage_account_name,
self.subscription_id,
)
return client
def cost_usage_source_is_reachable(self, credentials, data_source):
"""
Verify that the cost usage report source is reachable by Koku.
Implemented by provider specific class. An account validation and
connectivity check is to be done.
Args:
credentials (dict): Azure credentials dict
example: {'subscription_id': 'f695f74f-36a4-4112-9fe6-74415fac75a2',
'tenant_id': '319d4d72-7ddc-45d0-9d63-a2db0a36e048',
'client_id': 'ce26bd50-2e5a-4eb7-9504-a05a79568e25',
'client_secret': 'abc123' }
data_source (dict): Identifier of the cost usage report source
example: { 'resource_group': 'My Resource Group 1',
'storage_account': 'My Storage Account 2'
Returns:
None
Raises:
ValidationError: Error string
"""
key = "azure.error"
azure_service = None
if not (isinstance(credentials, dict)
and isinstance(data_source, dict)):
message = "Resource group and/or Storage account must be a dict"
raise ValidationError(error_obj(key, message))
resource_group = data_source.get("resource_group")
storage_account = data_source.get("storage_account")
subscription_id = credentials.get("subscription_id")
self._verify_patch_entries(subscription_id, resource_group,
storage_account)
try:
azure_service = AzureService(**credentials,
resource_group_name=resource_group,
storage_account_name=storage_account)
azure_client = AzureClientFactory(**credentials)
storage_accounts = azure_client.storage_client.storage_accounts
storage_account = storage_accounts.get_properties(
resource_group, storage_account)
if azure_service and not azure_service.describe_cost_management_exports(
):
key = ProviderErrors.AZURE_NO_REPORT_FOUND
message = ProviderErrors.AZURE_MISSING_EXPORT_MESSAGE
raise ValidationError(error_obj(key, message))
except AzureCostReportNotFound as costreport_err:
key = ProviderErrors.AZURE_BILLING_SOURCE_NOT_FOUND
raise ValidationError(error_obj(key, str(costreport_err)))
except (AdalError, AzureException, AzureServiceError, ClientException,
TypeError) as exc:
key = ProviderErrors.AZURE_CLIENT_ERROR
raise ValidationError(error_obj(key, str(exc)))
return True