def test_download_report_missing_bucket(self, mock_stats, fake_session):
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime('%Y%m%dT000000.000Z')
expected_assembly_id = '882083b7-ea62-4aab-aa6a-f0d08d65ee2b'
input_key = f'/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz'
mock_manifest = {
'assemblyId': expected_assembly_id,
'billingPeriod': {
'start': fake_report_date_str
},
'reportKeys': [input_key]
}
with patch.object(AWSReportDownloader,
'_get_manifest',
return_value=mock_manifest):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(self.fake_customer_name,
self.auth_credential,
self.fake_bucket_name,
'AWS', 2)
AWSReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_id': 2
})
report_downloader.download_report(fake_report_date)
def setUp(self):
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = 'ocp-report'
self.cluster_id = 'my-ocp-cluster-1'
report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id,
'20180901-20181001')
os.makedirs(report_path, exist_ok=True)
test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = './koku/masu/test/data/ocp/manifest.json'
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(self.fake_customer_name,
self.cluster_id, None, 'OCP',
self.ocp_provider_id)
self.ocp_report_downloader = OCPReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.cluster_id,
'bucket': None,
'provider_id': 1,
})
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz")
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS_LOCAL,
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": self.fake_bucket_name,
"provider_uuid": self.aws_provider_uuid,
}
)
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz")
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.credentials = {"role_arn": self.fake_auth_credential}
self.data_source = {"bucket": self.fake_bucket_name}
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source=self.data_source,
provider_type=Provider.PROVIDER_AWS_LOCAL,
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": self.data_source,
"provider_uuid": self.aws_provider_uuid,
})
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open(
"./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
fake_data_source = {"bucket": fake_bucket}
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_downloader = ReportDownloader(
self.fake_customer_name,
self.credentials,
fake_data_source,
Provider.PROVIDER_AWS_LOCAL,
self.aws_provider_uuid,
)
# Names from test report .gz file
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file":
"./koku/masu/test/data/test_local_bucket.tar.gz",
}
report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name,
"aws-local")
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def setUp(self, fake_session):
"""Set up shared variables."""
super().setUp()
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
cache_key=self.fake.word(),
)
self.aws_report_downloader = AWSReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.auth_credential,
"bucket": self.fake_bucket_name,
"report_name": self.fake_report_name,
"provider_uuid": self.aws_provider_uuid,
})
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open('./koku/masu/test/data/test_local_bucket.tar.gz')
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()),
return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type='AWS-local',
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': self.fake_bucket_name,
'provider_uuid': self.aws_provider_uuid,
}
)
def test_download_report_missing_bucket(self, mock_stats, fake_session):
"""Test download fails when bucket is missing."""
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime("%Y%m%dT000000.000Z")
expected_assembly_id = "882083b7-ea62-4aab-aa6a-f0d08d65ee2b"
input_key = f"/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz"
mock_manifest = {
"assemblyId": expected_assembly_id,
"billingPeriod": {
"start": fake_report_date_str
},
"reportKeys": [input_key],
}
with patch.object(AWSReportDownloader,
"_get_manifest",
return_value=("", mock_manifest)):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source=self.data_source,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
)
report_context = {
"date": fake_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": "/my/file",
}
report_downloader.download_report(report_context)
def test_download_report_missing_bucket(self, mock_stats, fake_session):
"""Test download fails when bucket is missing."""
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime("%Y%m%dT000000.000Z")
expected_assembly_id = "882083b7-ea62-4aab-aa6a-f0d08d65ee2b"
input_key = f"/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz"
mock_manifest = {
"assemblyId": expected_assembly_id,
"billingPeriod": {"start": fake_report_date_str},
"reportKeys": [input_key],
}
with patch.object(AWSReportDownloader, "_get_manifest", return_value=("", mock_manifest)):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
)
AWSReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.auth_credential,
"bucket": self.fake_bucket_name,
"report_name": self.fake_report_name,
"provider_uuid": self.aws_provider_uuid,
}
)
report_downloader.download_report(fake_report_date)
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = 'testcustomer'
self.local_storage = tempfile.mkdtemp()
self.container_name = 'my_container'
self.directory = 'dir'
self.export_name = 'myexport'
self.date_range = '20190801-20190831'
self.fake_auth_credential = {"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh"
}
}
self.fake_bucket_name = {
"resource_group": {
"export_name": self.export_name,
"directory": self.directory
},
"storage_account": {
"local_dir": self.local_storage,
"container": self.container_name
}
}
test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
local_dir = '{}/{}/{}/{}/{}'.format(self.local_storage,
self.container_name,
self.directory,
self.export_name,
self.date_range)
os.makedirs(local_dir)
self.csv_file_name = test_report.split('/')[-1]
self.csv_key = f'{local_dir}/{self.csv_file_name}'
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()),
return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type='AZURE-local',
provider_uuid=self.azure_provider_uuid,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.customer_name,
'auth_credential': self.fake_auth_credential,
'billing_source': self.fake_bucket_name,
'bucket': self.fake_bucket_name,
'provider_uuid': self.azure_provider_uuid,
}
)
def _get_report_files(task, customer_name, authentication, billing_source,
provider_type, provider_uuid, report_month):
"""
Task to download a Report.
Args:
task (Object): Bound celery task.
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
provider_uuid (String): Provider uuid.
report_month (DateTime): Month for report to download.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
month_string = report_month.strftime("%B %Y")
log_statement = (f"Downloading report for:\n"
f" schema_name: {customer_name}\n"
f" provider: {provider_type}\n"
f" account (provider uuid): {provider_uuid}\n"
f" report_month: {month_string}")
LOG.info(log_statement)
try:
disk = psutil.disk_usage(Config.PVC_DIR)
disk_msg = f"Available disk space: {disk.free} bytes ({100 - disk.percent}%)"
except OSError:
disk_msg = f"Unable to find available disk space. {Config.PVC_DIR} does not exist"
LOG.info(disk_msg)
reports = None
try:
downloader = ReportDownloader(
task=task,
customer_name=customer_name,
access_credential=authentication,
report_source=billing_source,
provider_type=provider_type,
provider_uuid=provider_uuid,
report_name=None,
)
reports = downloader.download_report(report_month)
except (MasuProcessingError, MasuProviderError,
ReportDownloaderError) as err:
worker_stats.REPORT_FILE_DOWNLOAD_ERROR_COUNTER.labels(
provider_type=provider_type).inc()
LOG.error(str(err))
with ProviderStatus(provider_uuid) as status:
status.set_error(error=err)
raise err
with ProviderStatus(provider_uuid) as status:
status.set_status(ProviderStatusCode.READY)
return reports
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = "testcustomer"
self.local_storage = tempfile.mkdtemp()
self.container_name = "my_container"
self.directory = "dir"
self.export_name = "myexport"
self.date_range = "20190801-20190831"
self.fake_auth_credential = {
"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh",
}
}
self.fake_bucket_name = {
"resource_group": {
"export_name": self.export_name,
"directory": self.directory
},
"storage_account": {
"local_dir": self.local_storage,
"container": self.container_name
},
}
test_report = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
local_dir = "{}/{}/{}/{}/{}".format(self.local_storage,
self.container_name,
self.directory, self.export_name,
self.date_range)
os.makedirs(local_dir)
self.csv_file_name = test_report.split("/")[-1]
self.csv_key = f"{local_dir}/{self.csv_file_name}"
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(
customer_name=self.customer_name,
credentials=self.fake_auth_credential,
data_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AZURE_LOCAL,
provider_uuid=self.azure_provider_uuid,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
"customer_name": self.customer_name,
"credentials": self.fake_auth_credential,
"data_source": self.fake_bucket_name,
"bucket": self.fake_bucket_name, # TODO: bucket?
"provider_uuid": self.azure_provider_uuid,
})
def test_get_reports_error(self, fake_downloader):
"""Test get_reports function with error."""
downloader = ReportDownloader(customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=AMAZON_WEB_SERVICES,
provider_id=1)
with patch.object(AWSReportDownloader, 'get_report_context_for_date', side_effect=Exception('some error')):
with self.assertRaises(ReportDownloaderError):
downloader.get_reports()
def _get_report_files(customer_name,
authentication,
billing_source,
provider_type,
report_name=None):
"""
Task to download a Report.
Note that report_name will be not optional once Koku can specify
what report we should download.
Args:
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
report_name (String): Name of the cost usage report to download.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
stmt = ('Downloading report for'
' credential: {},'
' source: {},'
' customer_name: {},'
' provider: {}')
log_statement = stmt.format(authentication, billing_source, customer_name,
provider_type)
LOG.info(log_statement)
try:
disk = psutil.disk_usage(Config.TMP_DIR)
disk_msg = 'Avaiable disk space: {} bytes ({}%)'.format(
disk.free, 100 - disk.percent)
except OSError:
disk_msg = 'Unable to find avaiable disk space. {} does not exist'.format(
Config.TMP_DIR)
LOG.info(disk_msg)
try:
downloader = ReportDownloader(customer_name=customer_name,
access_credential=authentication,
report_source=billing_source,
provider_type=provider_type,
report_name=report_name)
return downloader.get_current_report()
except (MasuProcessingError, MasuProviderError,
ReportDownloaderError) as err:
LOG.error(str(err))
raise err
def test_get_current_report(self, fake_downloader):
"""Test get_current_report function."""
downloader = ReportDownloader(customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=AMAZON_WEB_SERVICES)
with patch.object(AWSReportDownloader,
'download_current_report',
return_value=self.file_list):
files = downloader.get_current_report()
self.assertEqual(len(files), len(self.file_list))
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
cache_key=self.fake.word(),
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.cluster_id,
"bucket": None,
"provider_uuid": self.ocp_provider_uuid,
"cache_key": self.fake.word(),
})
def setUp(self, fake_session):
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(self.fake_customer_name,
self.auth_credential,
self.fake_bucket_name, 'AWS',
1)
self.aws_report_downloader = AWSReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_id': 1
})
def setUp(self):
"""Set up each test."""
super().setUp()
self.start_date = datetime(year=2020, month=11, day=8).date()
self.etag = "30c31bca571d9b7f3b2c8459dd8bc34a"
self.invoice = "202011"
test_report = f"./koku/masu/test/data/gcp/{self.invoice}_{self.etag}_2020-11-08:2020-11-11.csv"
self.local_storage = tempfile.mkdtemp()
local_dir = f"{self.local_storage}/{self.etag}"
os.makedirs(local_dir)
self.csv_file_name = test_report.split("/")[-1]
self.csv_file_path = f"{local_dir}/{self.csv_file_name}"
shutil.copy2(test_report, self.csv_file_path)
self.credentials = {"project_id": "test-project"}
self.data_source = {"table_id": "test-id", "dataset": "test-database", "local_dir": local_dir}
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source=self.data_source,
provider_type=Provider.PROVIDER_GCP_LOCAL,
provider_uuid=self.gcp_provider_uuid,
)
self.gcp_local_report_downloader = GCPLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": self.data_source,
"provider_uuid": self.gcp_provider_uuid,
}
)
def test_download_missing_month(self):
"""Test to verify that downloading a non-existant month throws proper exception."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz')
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=7, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
report_downloader = ReportDownloader(self.fake_customer_name,
self.fake_auth_credential,
fake_bucket, 'AWS-local', 1)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.fake_customer_name,
'aws-local')
self.assertFalse(os.path.isdir(expected_path))
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
self.credentials = {"cluster_id": self.cluster_id}
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source={},
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {},
"provider_uuid": self.ocp_provider_uuid,
})
def test_initializer_downloader_exception(self, fake_downloader):
"""Test to initializer where _set_downloader throws exception"""
with self.assertRaises(ReportDownloaderError):
ReportDownloader(customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=AMAZON_WEB_SERVICES)
def test_initializer(self, fake_downloader):
"""Test to initializer"""
downloader = ReportDownloader(customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=AMAZON_WEB_SERVICES)
self.assertIsNotNone(downloader._downloader)
def test_initializer_ocp(self, fake_downloader):
"""Test to initializer for OCP downloader"""
downloader = ReportDownloader(customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=OCP_LOCAL_SERVICE_PROVIDER,
provider_id=1)
self.assertIsNotNone(downloader._downloader)
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz')
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
report_downloader = ReportDownloader(self.fake_customer_name,
self.fake_auth_credential,
fake_bucket, 'AWS-local', 1)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.fake_customer_name,
'aws-local')
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def setUp(self):
"""Set up each test."""
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket.tar.gz')
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(self.fake_customer_name,
self.fake_auth_credential,
self.fake_bucket_name,
'AWS-local', 1)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': self.fake_bucket_name,
'provider_id': 1
})
def test_invalid_provider_type(self):
"""Test that error is thrown with invalid account source."""
with self.assertRaises(ReportDownloaderError):
ReportDownloader(customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type='unknown')
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = 'ocp-report'
self.cluster_id = 'my-ocp-cluster-1'
report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id,
'20180901-20181001')
os.makedirs(report_path, exist_ok=True)
test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91' \
'-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = './koku/masu/test/data/ocp/manifest.json'
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type='OCP',
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.cluster_id,
'bucket': None,
'provider_uuid': self.ocp_provider_uuid,
})
def test_download_missing_month(self):
"""Test to verify that downloading a non-existant month throws proper exception."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=7, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
report_downloader = ReportDownloader(
self.mock_task,
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
Provider.PROVIDER_AWS_LOCAL,
1,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local")
self.assertFalse(os.path.isdir(expected_path))
def test_initializer_azure(self, fake_downloader):
"""Test to initializer for Azure downloader"""
downloader = ReportDownloader(
customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=AZURE,
provider_id=self.azure_provider_id,
)
self.assertIsNotNone(downloader._downloader)
def test_initializer_ocp(self, fake_downloader):
"""Test to initializer"""
downloader = ReportDownloader(
customer_name='customer name',
access_credential=self.fake_creds,
report_source='hereiam',
report_name='bestreport',
provider_type=OPENSHIFT_CONTAINER_PLATFORM,
provider_id=self.ocp_provider_id,
)
self.assertIsNotNone(downloader._downloader)
