def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = 'testcustomer'
self.local_storage = tempfile.mkdtemp()
self.container_name = 'my_container'
self.directory = 'dir'
self.export_name = 'myexport'
self.date_range = '20190801-20190831'
self.fake_auth_credential = {"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh"
}
}
self.fake_bucket_name = {
"resource_group": {
"export_name": self.export_name,
"directory": self.directory
},
"storage_account": {
"local_dir": self.local_storage,
"container": self.container_name
}
}
test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
local_dir = '{}/{}/{}/{}/{}'.format(self.local_storage,
self.container_name,
self.directory,
self.export_name,
self.date_range)
os.makedirs(local_dir)
self.csv_file_name = test_report.split('/')[-1]
self.csv_key = f'{local_dir}/{self.csv_file_name}'
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()),
return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type='AZURE-local',
provider_uuid=self.azure_provider_uuid,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.customer_name,
'auth_credential': self.fake_auth_credential,
'billing_source': self.fake_bucket_name,
'bucket': self.fake_bucket_name,
'provider_uuid': self.azure_provider_uuid,
}
)
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = "testcustomer"
self.local_storage = tempfile.mkdtemp()
self.container_name = "my_container"
self.directory = "dir"
self.export_name = "myexport"
self.date_range = "20190801-20190831"
self.fake_auth_credential = {
"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh",
}
}
self.fake_bucket_name = {
"resource_group": {
"export_name": self.export_name,
"directory": self.directory
},
"storage_account": {
"local_dir": self.local_storage,
"container": self.container_name
},
}
test_report = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
local_dir = "{}/{}/{}/{}/{}".format(self.local_storage,
self.container_name,
self.directory, self.export_name,
self.date_range)
os.makedirs(local_dir)
self.csv_file_name = test_report.split("/")[-1]
self.csv_key = f"{local_dir}/{self.csv_file_name}"
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(
customer_name=self.customer_name,
credentials=self.fake_auth_credential,
data_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AZURE_LOCAL,
provider_uuid=self.azure_provider_uuid,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
"customer_name": self.customer_name,
"credentials": self.fake_auth_credential,
"data_source": self.fake_bucket_name,
"bucket": self.fake_bucket_name, # TODO: bucket?
"provider_uuid": self.azure_provider_uuid,
})
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == AMAZON_WEB_SERVICES:
return AWSReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AWS_LOCAL_SERVICE_PROVIDER:
return AWSLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE:
return AzureReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE_LOCAL_SERVICE_PROVIDER:
return AzureLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == OPENSHIFT_CONTAINER_PLATFORM:
return OCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == GCP:
return GCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
return None
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == Provider.PROVIDER_AWS:
return AWSReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AWS_LOCAL:
return AWSLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE:
return AzureReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE_LOCAL:
return AzureLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_OCP:
return OCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_GCP:
return GCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
return None
class AzureLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the AZURE-Local Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = 'testcustomer'
self.local_storage = tempfile.mkdtemp()
self.container_name = 'my_container'
self.directory = 'dir'
self.export_name = 'myexport'
self.date_range = '20190801-20190831'
self.fake_auth_credential = {
"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh"
}
}
self.fake_bucket_name = {
"resource_group": {
"export_name": self.export_name,
"directory": self.directory
},
"storage_account": {
"local_dir": self.local_storage,
"container": self.container_name
}
}
test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
local_dir = '{}/{}/{}/{}/{}'.format(self.local_storage,
self.container_name,
self.directory, self.export_name,
self.date_range)
os.makedirs(local_dir)
self.csv_file_name = test_report.split('/')[-1]
self.csv_key = f'{local_dir}/{self.csv_file_name}'
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(
self.customer_name,
self.fake_auth_credential,
self.fake_bucket_name,
'AZURE-local',
self.azure_provider_id,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
'customer_name': self.customer_name,
'auth_credential': self.fake_auth_credential,
'billing_source': self.fake_bucket_name,
'bucket': self.fake_bucket_name,
'provider_id': self.azure_provider_id,
})
def tearDown(self):
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.local_storage)
def test_initializer(self):
"""Test the Azure-Local initializer"""
self.assertIsNotNone(self.report_downloader)
def test_download_file(self):
"""Test Azure-Local report download"""
expected_full_path = '{}/{}/azure/{}/{}'.format(
Config.TMP_DIR, self.customer_name.replace(' ', '_'),
self.container_name, self.csv_file_name)
full_file_path, etag = self.azure_local_report_downloader.download_file(
self.csv_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertIsNotNone(etag)
# Download a second time, verify etag is returned
full_file_path, second_run_etag = self.azure_local_report_downloader.download_file(
self.csv_key)
self.assertEqual(etag, second_run_etag)
self.assertEqual(full_file_path, expected_full_path)
def test_download_report(self):
"""Test the top level Azure-Local download_report."""
test_report_date = datetime.datetime(year=2019, month=8, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.customer_name,
'azure')
self.assertTrue(os.path.isdir(expected_path))
class AzureLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the AZURE-Local Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = "testcustomer"
self.local_storage = tempfile.mkdtemp()
self.container_name = "my_container"
self.directory = "dir"
self.export_name = "myexport"
self.date_range = "20190801-20190831"
self.fake_auth_credential = {
"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh",
}
}
self.fake_bucket_name = {
"resource_group": {"export_name": self.export_name, "directory": self.directory},
"storage_account": {"local_dir": self.local_storage, "container": self.container_name},
}
test_report = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
local_dir = "{}/{}/{}/{}/{}".format(
self.local_storage, self.container_name, self.directory, self.export_name, self.date_range
)
os.makedirs(local_dir)
self.csv_file_name = test_report.split("/")[-1]
self.csv_key = f"{local_dir}/{self.csv_file_name}"
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AZURE_LOCAL,
provider_uuid=self.azure_provider_uuid,
cache_key=self.fake.word(),
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.customer_name,
"auth_credential": self.fake_auth_credential,
"billing_source": self.fake_bucket_name,
"bucket": self.fake_bucket_name,
"provider_uuid": self.azure_provider_uuid,
"cache_key": self.fake.word(),
}
)
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.local_storage)
def test_initializer(self):
"""Test the Azure-Local initializer."""
self.assertIsNotNone(self.report_downloader)
def test_download_file(self):
"""Test Azure-Local report download."""
expected_full_path = "{}/{}/azure/{}/{}".format(
Config.TMP_DIR, self.customer_name.replace(" ", "_"), self.container_name, self.csv_file_name
)
full_file_path, etag = self.azure_local_report_downloader.download_file(self.csv_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertIsNotNone(etag)
# Download a second time, verify etag is returned
full_file_path, second_run_etag = self.azure_local_report_downloader.download_file(self.csv_key)
self.assertEqual(etag, second_run_etag)
self.assertEqual(full_file_path, expected_full_path)
def test_download_report(self):
"""Test the top level Azure-Local download_report."""
test_report_date = datetime.datetime(year=2019, month=8, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.customer_name, "azure")
self.assertTrue(os.path.isdir(expected_path))
class AzureLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the AZURE-Local Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = 'testcustomer'
self.local_storage = tempfile.mkdtemp()
self.container_name = 'my_container'
self.directory = 'dir'
self.export_name = 'myexport'
self.date_range = '20190801-20190831'
self.fake_auth_credential = {
'credentials': {
'subscription_id': '2639de71-ca37-4a17-a104-17665a51e7fc',
'tenant_id': 'ae4f8f55-f1a8-4080-9aa8-10779e4113f7',
'client_id': 'd6b607d7-d07a-4ca0-b81d-39631f7323aa',
'client_secret': 'ahhhhh',
}
}
self.fake_bucket_name = {
'resource_group': {'export_name': self.export_name, 'directory': self.directory},
'storage_account': {'local_dir': self.local_storage, 'container': self.container_name},
}
test_report = (
'./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
)
local_dir = '{}/{}/{}/{}/{}'.format(
self.local_storage,
self.container_name,
self.directory,
self.export_name,
self.date_range,
)
os.makedirs(local_dir)
self.csv_file_name = test_report.split('/')[-1]
self.csv_key = f'{local_dir}/{self.csv_file_name}'
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AZURE_LOCAL,
provider_uuid=self.azure_provider_uuid,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.customer_name,
'auth_credential': self.fake_auth_credential,
'billing_source': self.fake_bucket_name,
'bucket': self.fake_bucket_name,
'provider_uuid': self.azure_provider_uuid,
}
)
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.local_storage)
def test_initializer(self):
"""Test the Azure-Local initializer."""
self.assertIsNotNone(self.report_downloader)
def test_download_file(self):
"""Test Azure-Local report download."""
expected_full_path = '{}/{}/azure/{}/{}'.format(
Config.TMP_DIR,
self.customer_name.replace(' ', '_'),
self.container_name,
self.csv_file_name,
)
full_file_path, etag = self.azure_local_report_downloader.download_file(self.csv_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertIsNotNone(etag)
# Download a second time, verify etag is returned
full_file_path, second_run_etag = self.azure_local_report_downloader.download_file(
self.csv_key
)
self.assertEqual(etag, second_run_etag)
self.assertEqual(full_file_path, expected_full_path)
def test_download_report(self):
"""Test the top level Azure-Local download_report."""
test_report_date = datetime.datetime(year=2019, month=8, day=7)
with patch.object(DateAccessor, 'today', return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.customer_name, 'azure')
self.assertTrue(os.path.isdir(expected_path))