def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz')
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
report_downloader = ReportDownloader(self.fake_customer_name,
self.fake_auth_credential,
fake_bucket, 'AWS-local', 1)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.fake_customer_name,
'aws-local')
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def _get_report_files(task, customer_name, authentication, billing_source,
provider_type, provider_uuid, report_month):
"""
Task to download a Report.
Args:
task (Object): Bound celery task.
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
provider_uuid (String): Provider uuid.
report_month (DateTime): Month for report to download.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
month_string = report_month.strftime('%B %Y')
log_statement = (f'Downloading report for:\n'
f' schema_name: {customer_name}\n'
f' provider: {provider_type}\n'
f' account (provider uuid): {provider_uuid}\n'
f' report_month: {month_string}')
LOG.info(log_statement)
try:
disk = psutil.disk_usage(Config.PVC_DIR)
disk_msg = f'Available disk space: {disk.free} bytes ({100 - disk.percent}%)'
except OSError:
disk_msg = f'Unable to find available disk space. {Config.PVC_DIR} does not exist'
LOG.info(disk_msg)
reports = None
try:
downloader = ReportDownloader(task=task,
customer_name=customer_name,
access_credential=authentication,
report_source=billing_source,
provider_type=provider_type,
provider_uuid=provider_uuid,
report_name=None)
reports = downloader.download_report(report_month)
except (MasuProcessingError, MasuProviderError,
ReportDownloaderError) as err:
worker_stats.REPORT_FILE_DOWNLOAD_ERROR_COUNTER.labels(
provider_type=provider_type).inc()
LOG.error(str(err))
with ProviderStatus(provider_uuid) as status:
status.set_error(error=err)
raise err
with ProviderStatus(provider_uuid) as status:
status.set_status(ProviderStatusCode.READY)
return reports
def test_download_missing_month(self):
"""Test to verify that downloading a non-existant month throws proper exception."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=7, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
report_downloader = ReportDownloader(
self.mock_task,
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
Provider.PROVIDER_AWS_LOCAL,
1,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local")
self.assertFalse(os.path.isdir(expected_path))
def test_download_missing_month(self):
"""Test to verify that downloading a non-existant month throws proper exception."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz')
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=7, day=7)
with patch.object(DateAccessor, 'today', return_value=test_report_date):
report_downloader = ReportDownloader(
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
'AWS-local',
1,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(
DATA_DIR, self.fake_customer_name, 'aws-local'
)
self.assertFalse(os.path.isdir(expected_path))
def test_download_report_missing_bucket(self, mock_stats, fake_session):
"""Test download fails when bucket is missing."""
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime("%Y%m%dT000000.000Z")
expected_assembly_id = "882083b7-ea62-4aab-aa6a-f0d08d65ee2b"
input_key = f"/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz"
mock_manifest = {
"assemblyId": expected_assembly_id,
"billingPeriod": {
"start": fake_report_date_str
},
"reportKeys": [input_key],
}
with patch.object(AWSReportDownloader,
"_get_manifest",
return_value=("", mock_manifest)):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
)
AWSReportDownloader(
**{
"customer_name": self.fake_customer_name,
"auth_credential": self.auth_credential,
"bucket": self.fake_bucket_name,
"report_name": self.fake_report_name,
"provider_uuid": self.aws_provider_uuid,
})
report_context = {
"date": fake_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": "/my/file",
}
report_downloader.download_report(report_context)
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
report_downloader = ReportDownloader(
self.mock_task,
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
Provider.PROVIDER_AWS_LOCAL,
self.aws_provider_uuid,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local")
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def test_download_report_missing_bucket(self, mock_stats, fake_session):
"""Test download fails when bucket is missing."""
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime('%Y%m%dT000000.000Z')
expected_assembly_id = '882083b7-ea62-4aab-aa6a-f0d08d65ee2b'
input_key = f'/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz'
mock_manifest = {
'assemblyId': expected_assembly_id,
'billingPeriod': {
'start': fake_report_date_str
},
'reportKeys': [input_key],
}
with patch.object(AWSReportDownloader,
'_get_manifest',
return_value=('', mock_manifest)):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
)
AWSReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_uuid': self.aws_provider_uuid,
})
report_downloader.download_report(fake_report_date)
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = 'ocp-report'
self.cluster_id = 'my-ocp-cluster-1'
report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id,
'20180901-20181001')
os.makedirs(report_path, exist_ok=True)
test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = './koku/masu/test/data/ocp/manifest.json'
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type='OCP',
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.cluster_id,
'bucket': None,
'provider_uuid': self.ocp_provider_uuid,
})
def tearDown(self):
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
def test_download_bucket(self):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(Config.TMP_DIR,
self.fake_customer_name, 'ocp')
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
reports = []
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
os.remove(self.test_file_path)
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
reports = []
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
# Create .txt file
txt_file_path = '{}/{}'.format(
os.path.dirname(self.test_file_path), 'report.txt')
open(txt_file_path, 'a').close()
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_source_directory_missing(self):
"""Test to verify that basic report downloading when source directory doesn't exist."""
reports = []
# Set current date to a day that is outside of the test file's date range.
test_report_date = datetime(year=2018, month=10, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
class AzureLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the AZURE-Local Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = 'testcustomer'
self.local_storage = tempfile.mkdtemp()
self.container_name = 'my_container'
self.directory = 'dir'
self.export_name = 'myexport'
self.date_range = '20190801-20190831'
self.fake_auth_credential = {
'credentials': {
'subscription_id': '2639de71-ca37-4a17-a104-17665a51e7fc',
'tenant_id': 'ae4f8f55-f1a8-4080-9aa8-10779e4113f7',
'client_id': 'd6b607d7-d07a-4ca0-b81d-39631f7323aa',
'client_secret': 'ahhhhh',
}
}
self.fake_bucket_name = {
'resource_group': {'export_name': self.export_name, 'directory': self.directory},
'storage_account': {'local_dir': self.local_storage, 'container': self.container_name},
}
test_report = (
'./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
)
local_dir = '{}/{}/{}/{}/{}'.format(
self.local_storage,
self.container_name,
self.directory,
self.export_name,
self.date_range,
)
os.makedirs(local_dir)
self.csv_file_name = test_report.split('/')[-1]
self.csv_key = f'{local_dir}/{self.csv_file_name}'
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AZURE_LOCAL,
provider_uuid=self.azure_provider_uuid,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.customer_name,
'auth_credential': self.fake_auth_credential,
'billing_source': self.fake_bucket_name,
'bucket': self.fake_bucket_name,
'provider_uuid': self.azure_provider_uuid,
}
)
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.local_storage)
def test_initializer(self):
"""Test the Azure-Local initializer."""
self.assertIsNotNone(self.report_downloader)
def test_download_file(self):
"""Test Azure-Local report download."""
expected_full_path = '{}/{}/azure/{}/{}'.format(
Config.TMP_DIR,
self.customer_name.replace(' ', '_'),
self.container_name,
self.csv_file_name,
)
full_file_path, etag = self.azure_local_report_downloader.download_file(self.csv_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertIsNotNone(etag)
# Download a second time, verify etag is returned
full_file_path, second_run_etag = self.azure_local_report_downloader.download_file(
self.csv_key
)
self.assertEqual(etag, second_run_etag)
self.assertEqual(full_file_path, expected_full_path)
def test_download_report(self):
"""Test the top level Azure-Local download_report."""
test_report_date = datetime.datetime(year=2019, month=8, day=7)
with patch.object(DateAccessor, 'today', return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.customer_name, 'azure')
self.assertTrue(os.path.isdir(expected_path))
class AWSLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the Local Report Downloader."""
fake = Faker()
@classmethod
def setUpClass(cls):
"""Set up class variables."""
super().setUpClass()
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = "koku-local"
cls.fake_bucket_prefix = PREFIX
cls.selected_region = REGION
cls.fake_auth_credential = fake_arn(service="iam",
generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz")
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.credentials = {"role_arn": self.fake_auth_credential}
self.data_source = {"bucket": self.fake_bucket_name}
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source=self.data_source,
provider_type=Provider.PROVIDER_AWS_LOCAL,
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": self.data_source,
"provider_uuid": self.aws_provider_uuid,
})
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.fake_bucket_name)
@patch("masu.processor.parquet.parquet_report_processor.settings",
ENABLE_S3_ARCHIVING=True)
def test_download_bucket(self, _):
"""Test to verify that basic report downloading works."""
with patch("masu.processor.parquet.parquet_report_processor.Path"):
with patch("masu.processor.parquet.parquet_report_processor.pd"):
with patch(
"masu.processor.parquet.parquet_report_processor.open"
):
with patch(
"masu.processor.parquet.parquet_report_processor.copy_data_to_s3_bucket"
):
with patch(
"masu.processor.parquet.parquet_report_processor.ParquetReportProcessor."
"create_parquet_table"):
test_report_date = datetime(year=2018,
month=8,
day=7)
with patch.object(DateAccessor,
"today",
return_value=test_report_date):
report_context = {
"date":
test_report_date.date(),
"manifest_id":
1,
"comporession":
"GZIP",
"current_file":
"./koku/masu/test/data/test_local_bucket.tar.gz",
}
self.report_downloader.download_report(
report_context)
expected_path = "{}/{}/{}".format(
DATA_DIR, self.fake_customer_name, "aws-local")
self.assertTrue(os.path.isdir(expected_path))
def test_report_name_provided(self):
"""Test initializer when report_name is provided."""
report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": self.data_source,
"report_name": "awesome-report",
})
self.assertEqual(report_downloader.report_name, "awesome-report")
@patch("masu.processor.parquet.parquet_report_processor.settings",
ENABLE_S3_ARCHIVING=True)
def test_extract_names_no_prefix(self, _):
"""Test to extract the report and prefix names from a bucket with no prefix."""
with patch("masu.processor.parquet.parquet_report_processor.Path"):
with patch("masu.processor.parquet.parquet_report_processor.pd"):
with patch(
"masu.processor.parquet.parquet_report_processor.open"
):
with patch(
"masu.processor.parquet.parquet_report_processor.copy_data_to_s3_bucket"
):
with patch(
"masu.processor.parquet.parquet_report_processor.ParquetReportProcessor."
"create_parquet_table"):
report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": self.data_source,
})
self.assertEqual(report_downloader.report_name,
self.fake_report_name)
self.assertIsNone(report_downloader.report_prefix)
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open(
"./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
fake_data_source = {"bucket": fake_bucket}
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_downloader = ReportDownloader(
self.fake_customer_name,
self.credentials,
fake_data_source,
Provider.PROVIDER_AWS_LOCAL,
self.aws_provider_uuid,
)
# Names from test report .gz file
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file":
"./koku/masu/test/data/test_local_bucket.tar.gz",
}
report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name,
"aws-local")
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def test_extract_names_with_prefix(self):
"""Test to extract the report and prefix names from a bucket with prefix."""
bucket = tempfile.mkdtemp()
report_name = "report-name"
prefix_name = "prefix-name"
full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-20180901/"
os.makedirs(full_path)
report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {
"bucket": bucket
},
})
self.assertEqual(report_downloader.report_name, report_name)
self.assertEqual(report_downloader.report_prefix, prefix_name)
shutil.rmtree(full_path)
def test_extract_names_with_bad_path(self):
"""Test to extract the report and prefix names from a bad path."""
bucket = tempfile.mkdtemp()
report_name = "report-name"
prefix_name = "prefix-name"
full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-aaaaaaa/"
os.makedirs(full_path)
report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {
"bucket": bucket
},
})
self.assertIsNone(report_downloader.report_name)
self.assertIsNone(report_downloader.report_prefix)
shutil.rmtree(full_path)
def test_extract_names_with_incomplete_path(self):
"""Test to extract the report and prefix from a path where a CUR hasn't been generated yet."""
bucket = tempfile.mkdtemp()
report_downloader = AWSLocalReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {
"bucket": bucket
},
})
self.assertIsNone(report_downloader.report_name)
self.assertIsNone(report_downloader.report_prefix)
shutil.rmtree(bucket)
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger=
"masu.external.downloader.aws_local.aws_local_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.aws_local_report_downloader._remove_manifest_file("None")
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
@patch(
"masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._get_manifest"
)
def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
downloader = AWSLocalReportDownloader(
self.fake_customer_name,
self.credentials,
self.data_source,
provider_uuid=self.aws_provider_uuid)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = "GZIP"
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {
"start": start_str
},
},
DateAccessor().today(),
)
result = downloader.get_manifest_context_for_date(current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
@patch(
"masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._get_manifest"
)
def test_get_manifest_context_for_date_no_manifest(self, mock_manifest,
mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
downloader = AWSLocalReportDownloader(
self.fake_customer_name,
self.credentials,
self.data_source,
provider_uuid=self.aws_provider_uuid)
mock_manifest.return_value = ("", {
"reportKeys": []
}, DateAccessor().today())
result = downloader.get_manifest_context_for_date(current_month)
self.assertEqual(result, {})
class AzureLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the AZURE-Local Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = "testcustomer"
self.local_storage = tempfile.mkdtemp()
self.container_name = "my_container"
self.directory = "dir"
self.export_name = "myexport"
self.date_range = "20190801-20190831"
self.fake_auth_credential = {
"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh",
}
}
self.fake_bucket_name = {
"resource_group": {"export_name": self.export_name, "directory": self.directory},
"storage_account": {"local_dir": self.local_storage, "container": self.container_name},
}
test_report = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
local_dir = "{}/{}/{}/{}/{}".format(
self.local_storage, self.container_name, self.directory, self.export_name, self.date_range
)
os.makedirs(local_dir)
self.csv_file_name = test_report.split("/")[-1]
self.csv_key = f"{local_dir}/{self.csv_file_name}"
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AZURE_LOCAL,
provider_uuid=self.azure_provider_uuid,
cache_key=self.fake.word(),
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.customer_name,
"auth_credential": self.fake_auth_credential,
"billing_source": self.fake_bucket_name,
"bucket": self.fake_bucket_name,
"provider_uuid": self.azure_provider_uuid,
"cache_key": self.fake.word(),
}
)
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.local_storage)
def test_initializer(self):
"""Test the Azure-Local initializer."""
self.assertIsNotNone(self.report_downloader)
def test_download_file(self):
"""Test Azure-Local report download."""
expected_full_path = "{}/{}/azure/{}/{}".format(
Config.TMP_DIR, self.customer_name.replace(" ", "_"), self.container_name, self.csv_file_name
)
full_file_path, etag = self.azure_local_report_downloader.download_file(self.csv_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertIsNotNone(etag)
# Download a second time, verify etag is returned
full_file_path, second_run_etag = self.azure_local_report_downloader.download_file(self.csv_key)
self.assertEqual(etag, second_run_etag)
self.assertEqual(full_file_path, expected_full_path)
def test_download_report(self):
"""Test the top level Azure-Local download_report."""
test_report_date = datetime.datetime(year=2019, month=8, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.customer_name, "azure")
self.assertTrue(os.path.isdir(expected_path))
class AzureLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the AZURE-Local Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.customer_name = 'testcustomer'
self.local_storage = tempfile.mkdtemp()
self.container_name = 'my_container'
self.directory = 'dir'
self.export_name = 'myexport'
self.date_range = '20190801-20190831'
self.fake_auth_credential = {
"credentials": {
"subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc",
"tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7",
"client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa",
"client_secret": "ahhhhh"
}
}
self.fake_bucket_name = {
"resource_group": {
"export_name": self.export_name,
"directory": self.directory
},
"storage_account": {
"local_dir": self.local_storage,
"container": self.container_name
}
}
test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
local_dir = '{}/{}/{}/{}/{}'.format(self.local_storage,
self.container_name,
self.directory, self.export_name,
self.date_range)
os.makedirs(local_dir)
self.csv_file_name = test_report.split('/')[-1]
self.csv_key = f'{local_dir}/{self.csv_file_name}'
shutil.copy2(test_report, self.csv_key)
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(
self.customer_name,
self.fake_auth_credential,
self.fake_bucket_name,
'AZURE-local',
self.azure_provider_id,
)
self.azure_local_report_downloader = AzureLocalReportDownloader(
**{
'customer_name': self.customer_name,
'auth_credential': self.fake_auth_credential,
'billing_source': self.fake_bucket_name,
'bucket': self.fake_bucket_name,
'provider_id': self.azure_provider_id,
})
def tearDown(self):
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.local_storage)
def test_initializer(self):
"""Test the Azure-Local initializer"""
self.assertIsNotNone(self.report_downloader)
def test_download_file(self):
"""Test Azure-Local report download"""
expected_full_path = '{}/{}/azure/{}/{}'.format(
Config.TMP_DIR, self.customer_name.replace(' ', '_'),
self.container_name, self.csv_file_name)
full_file_path, etag = self.azure_local_report_downloader.download_file(
self.csv_key)
self.assertEqual(full_file_path, expected_full_path)
self.assertIsNotNone(etag)
# Download a second time, verify etag is returned
full_file_path, second_run_etag = self.azure_local_report_downloader.download_file(
self.csv_key)
self.assertEqual(etag, second_run_etag)
self.assertEqual(full_file_path, expected_full_path)
def test_download_report(self):
"""Test the top level Azure-Local download_report."""
test_report_date = datetime.datetime(year=2019, month=8, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(DATA_DIR, self.customer_name,
'azure')
self.assertTrue(os.path.isdir(expected_path))
class AWSLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the Local Report Downloader."""
fake = Faker()
@classmethod
def setUpClass(cls):
"""Set up class variables."""
super().setUpClass()
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = "koku-local"
cls.fake_bucket_prefix = PREFIX
cls.selected_region = REGION
cls.fake_auth_credential = fake_arn(service="iam", generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz")
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.fake_auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS_LOCAL,
provider_uuid=self.aws_provider_uuid,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": self.fake_bucket_name,
"provider_uuid": self.aws_provider_uuid,
}
)
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.fake_bucket_name)
def test_download_bucket(self):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local")
self.assertTrue(os.path.isdir(expected_path))
def test_report_name_provided(self):
"""Test initializer when report_name is provided."""
report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": self.fake_bucket_name,
"report_name": "awesome-report",
}
)
self.assertEqual(report_downloader.report_name, "awesome-report")
def test_extract_names_no_prefix(self):
"""Test to extract the report and prefix names from a bucket with no prefix."""
report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": self.fake_bucket_name,
}
)
self.assertEqual(report_downloader.report_name, self.fake_report_name)
self.assertIsNone(report_downloader.report_prefix)
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
report_downloader = ReportDownloader(
self.mock_task,
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
Provider.PROVIDER_AWS_LOCAL,
self.aws_provider_uuid,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local")
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def test_extract_names_with_prefix(self):
"""Test to extract the report and prefix names from a bucket with prefix."""
bucket = tempfile.mkdtemp()
report_name = "report-name"
prefix_name = "prefix-name"
full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-20180901/"
os.makedirs(full_path)
report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": bucket,
}
)
self.assertEqual(report_downloader.report_name, report_name)
self.assertEqual(report_downloader.report_prefix, prefix_name)
shutil.rmtree(full_path)
def test_extract_names_with_bad_path(self):
"""Test to extract the report and prefix names from a bad path."""
bucket = tempfile.mkdtemp()
report_name = "report-name"
prefix_name = "prefix-name"
full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-aaaaaaa/"
os.makedirs(full_path)
report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": bucket,
}
)
self.assertIsNone(report_downloader.report_name)
self.assertIsNone(report_downloader.report_prefix)
shutil.rmtree(full_path)
def test_extract_names_with_incomplete_path(self):
"""Test to extract the report and prefix from a path where a CUR hasn't been generated yet."""
bucket = tempfile.mkdtemp()
report_downloader = AWSLocalReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.fake_auth_credential,
"bucket": bucket,
}
)
self.assertIsNone(report_downloader.report_name)
self.assertIsNone(report_downloader.report_prefix)
shutil.rmtree(bucket)
def test_download_missing_month(self):
"""Test to verify that downloading a non-existant month throws proper exception."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz")
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=7, day=7)
with patch.object(DateAccessor, "today", return_value=test_report_date):
report_downloader = ReportDownloader(
self.mock_task,
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
Provider.PROVIDER_AWS_LOCAL,
1,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local")
self.assertFalse(os.path.isdir(expected_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.aws_local.aws_local_report_downloader", level="INFO"
) as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.aws_local_report_downloader._remove_manifest_file("None")
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg="The log is expected to start with 'INFO:' but instead was: " + captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
cache_key=self.fake.word(),
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.cluster_id,
"bucket": None,
"provider_uuid": self.ocp_provider_uuid,
"cache_key": self.fake.word(),
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
def test_download_bucket(self):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_source_directory_missing(self):
"""Test to verify that basic report downloading when source directory doesn't exist."""
reports = []
# Set current date to a day that is outside of the test file's date range.
test_report_date = datetime(year=2018, month=10, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def _get_report_files(
task,
customer_name,
authentication,
billing_source,
provider_type,
provider_uuid,
report_month,
cache_key,
report_context,
):
"""
Task to download a Report.
Args:
task (Object): Bound celery task.
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
provider_uuid (String): Provider uuid.
report_month (DateTime): Month for report to download.
cache_key (String): The provider specific task cache value.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
request_id = task.request.id
context = {"account": customer_name[4:], "provider_uuid": provider_uuid}
month_string = report_month.strftime("%B %Y")
report_context["date"] = report_month
log_statement = (f"Downloading report for:\n"
f" schema_name: {customer_name}\n"
f" provider: {provider_type}\n"
f" account (provider uuid): {provider_uuid}\n"
f" report_month: {month_string}")
LOG.info(log_json(request_id, log_statement, context))
try:
disk = psutil.disk_usage(Config.PVC_DIR)
disk_msg = f"Available disk space: {disk.free} bytes ({100 - disk.percent}%)"
except OSError:
disk_msg = f"Unable to find available disk space. {Config.PVC_DIR} does not exist"
LOG.info(log_json(request_id, disk_msg, context))
report = None
try:
downloader = ReportDownloader(
customer_name=customer_name,
credentials=authentication,
data_source=billing_source,
provider_type=provider_type,
provider_uuid=provider_uuid,
report_name=None,
account=customer_name[4:],
request_id=task.request.id,
)
report = downloader.download_report(report_context)
except (MasuProcessingError, MasuProviderError,
ReportDownloaderError) as err:
worker_stats.REPORT_FILE_DOWNLOAD_ERROR_COUNTER.labels(
provider_type=provider_type).inc()
WorkerCache().remove_task_from_cache(cache_key)
LOG.error(log_json(request_id, str(err), context))
raise err
return report
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
cache_key=self.fake.word(),
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.cluster_id,
"bucket": None,
"provider_uuid": self.ocp_provider_uuid,
"cache_key": self.fake.word(),
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
@patch("masu.util.aws.common.copy_data_to_s3_bucket", return_value=None)
def test_download_bucket(self, mock_copys3):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
with self.assertRaises(FileNotFoundError):
self.report_downloader.download_report(test_report_date)
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
with self.assertRaises(FileNotFoundError):
self.report_downloader.download_report(test_report_date)
def test_download_bucket_source_directory_missing(self):
"""Test to verify that basic report downloading when source directory doesn't exist."""
reports = []
# Set current date to a day that is outside of the test file's date range.
test_report_date = datetime(year=2018, month=10, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def test_divide_csv_daily(self):
"""Test the divide_csv_daily method."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_report = {
"interval_start": [
"2020-01-01 00:00:00 +UTC",
"2020-01-02 00:00:00 +UTC"
],
"persistentvolumeclaim_labels": ["label1", "label2"],
}
df = pd.DataFrame(data=mock_report)
mock_pd.read_csv.return_value = df
daily_files = divide_csv_daily(file_path, filename)
self.assertNotEqual([], daily_files)
self.assertEqual(len(daily_files), 2)
gen_files = [
"storage_usage.2020-01-01.csv",
"storage_usage.2020-01-02.csv"
]
expected = [{
"filename": gen_file,
"filepath": f"{td}/{gen_file}"
} for gen_file in gen_files]
for expected_item in expected:
self.assertIn(expected_item, daily_files)
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
self.credentials = {"cluster_id": self.cluster_id}
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source={},
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {},
"provider_uuid": self.ocp_provider_uuid,
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
@patch("masu.util.aws.common.copy_data_to_s3_bucket", return_value=None)
def test_download_bucket(self, mock_copys3):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def test_divide_csv_daily(self):
"""Test the divide_csv_daily method."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_report = {
"interval_start": [
"2020-01-01 00:00:00 +UTC",
"2020-01-02 00:00:00 +UTC"
],
"persistentvolumeclaim_labels": ["label1", "label2"],
}
df = pd.DataFrame(data=mock_report)
mock_pd.read_csv.return_value = df
daily_files = divide_csv_daily(file_path, filename)
self.assertNotEqual([], daily_files)
self.assertEqual(len(daily_files), 2)
gen_files = [
"storage_usage.2020-01-01.csv",
"storage_usage.2020-01-02.csv"
]
expected = [{
"filename": gen_file,
"filepath": f"{td}/{gen_file}"
} for gen_file in gen_files]
for expected_item in expected:
self.assertIn(expected_item, daily_files)
def test_divide_csv_daily_failure(self):
"""Test the divide_csv_daily method throw error on reading CSV."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
errorMsg = "CParserError: Error tokenizing data. C error: Expected 53 fields in line 1605634, saw 54"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_pd.read_csv.side_effect = Exception(errorMsg)
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.LOG.error"
) as mock_debug:
with self.assertRaises(Exception):
divide_csv_daily(file_path, filename)
mock_debug.assert_called_once_with(
f"File {file_path} could not be parsed. Reason: {errorMsg}"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.get_report_details"
)
def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
version = "5678"
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
"version": version,
}
self.assertIsNone(self.ocp_report_downloader.context.get("version"))
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
manifest_id = result.get("manifest_id")
manifest = ReportManifestDBAccessor().get_manifest_by_id(manifest_id)
self.assertEqual(manifest.operator_version, version)
self.assertEqual(self.ocp_report_downloader.context.get("version"),
version)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.get_report_details"
)
def test_get_manifest_context_new_info(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
version = "5678"
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
"version": version,
"certified": False,
"cluster_id": "4e009161-4f40-42c8-877c-3e59f6baea3d",
"cr_status": {
"clusterID": "4e009161-4f40-42c8-877c-3e59f6baea3d",
"clusterVersion": "stable-4.6",
"api_url": "https://cloud.redhat.com",
"authentication": {
"type": "token"
},
"packaging": {
"max_reports_to_store": 30,
"max_size_MB": 100
},
"upload": {
"ingress_path": "/api/ingress/v1/upload",
"upload": False
},
"operator_commit": "a09a5b21e55ce4a07fe31aa560650b538ec6de7c",
"prometheus": {
"error": "fake error"
},
"reports": {
"report_month":
"07",
"last_hour_queried":
"2021-07-28 11:00:00 - 2021-07-28 11:59:59"
},
"source": {
"sources_path": "/api/sources/v1.0/",
"name": "INSERT-SOURCE-NAME"
},
},
}
self.assertIsNone(self.ocp_report_downloader.context.get("version"))
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
manifest_id = result.get("manifest_id")
manifest = ReportManifestDBAccessor().get_manifest_by_id(manifest_id)
expected_errors = {"prometheus_error": "fake error"}
self.assertEqual(manifest.operator_version, version)
self.assertEqual(manifest.operator_certified, False)
self.assertEqual(manifest.operator_airgapped, True)
self.assertEqual(manifest.cluster_channel, "stable-4.6")
self.assertEqual(manifest.cluster_id,
"4e009161-4f40-42c8-877c-3e59f6baea3d")
self.assertEqual(manifest.operator_errors, expected_errors)
self.assertEqual(self.ocp_report_downloader.context.get("version"),
version)
@override_settings(ENABLE_S3_ARCHIVING=True)
@override_settings(ENABLE_PARQUET_PROCESSING=True)
@patch("masu.external.downloader.ocp.ocp_report_downloader.os")
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.copy_local_report_file_to_s3_bucket"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.divide_csv_daily")
def test_create_daily_archives(self, mock_divide, mock_s3_copy, mock_os):
"""Test that this method returns a file list."""
start_date = DateHelper().this_month_start
daily_files = [
{
"filename": "file_one",
"filepath": "path/to/file_one"
},
{
"filename": "file_two",
"filepath": "path/to/file_two"
},
]
expected_filenames = ["path/to/file_one", "path/to/file_two"]
mock_divide.return_value = daily_files
file_name = "file"
file_path = "path"
result = create_daily_archives(1, "10001", self.ocp_provider_uuid,
file_name, file_path, 1, start_date)
self.assertEqual(result, expected_filenames)
context = {"version": "1"}
expected = [file_path]
result = create_daily_archives(1,
"10001",
self.ocp_provider_uuid,
"file",
"path",
1,
start_date,
context=context)
self.assertEqual(result, expected)
@patch("masu.external.downloader.ocp.ocp_report_downloader.LOG")
def test_get_report_for_verify_tracing_id(self, log_mock):
self.ocp_report_downloader.tracing_id = "1111-2222-4444-5555"
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
self.ocp_report_downloader.get_report_for(current_month)
call_args = log_mock.debug.call_args[0][0]
self.assertTrue("Looking for cluster" in call_args.get("message"))
self.assertEqual(call_args.get("tracing_id"),
self.ocp_report_downloader.tracing_id)
call_args = log_mock.info.call_args[0][0]
self.assertTrue("manifest found:" in call_args.get("message"))
self.assertEqual(call_args.get("tracing_id"),
self.ocp_report_downloader.tracing_id)
class AWSReportDownloaderTest(MasuTestCase):
"""Test Cases for the AWS S3 functions."""
fake = Faker()
@classmethod
def setUpClass(cls):
"""Set up shared class variables."""
super().setUpClass()
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = REPORT
cls.fake_bucket_prefix = PREFIX
cls.fake_bucket_name = BUCKET
cls.selected_region = REGION
cls.auth_credential = fake_arn(service="iam", generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def setUp(self, fake_session):
"""Set up shared variables."""
super().setUp()
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
)
self.aws_report_downloader = AWSReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.auth_credential,
"bucket": self.fake_bucket_name,
"report_name": self.fake_report_name,
"provider_uuid": self.aws_provider_uuid,
}
)
def tearDown(self):
"""Remove test generated data."""
shutil.rmtree(DATA_DIR, ignore_errors=True)
@patch("masu.external.downloader.aws.aws_report_downloader.boto3.resource")
@patch(
"masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.download_file",
return_value=("mock_file_name", None),
)
def test_download_bucket(self, mock_boto_resource, mock_download_file):
"""Test download bucket method."""
mock_resource = Mock()
mock_bucket = Mock()
mock_bucket.objects.all.return_value = []
mock_resource.Bucket.return_value = mock_bucket
out = self.aws_report_downloader.download_bucket()
expected_files = []
self.assertEqual(out, expected_files)
@patch(
"masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.download_file",
side_effect=mock_download_file_error,
)
def test_download_report_missing_manifest(self, mock_download_file):
"""Test download fails when manifest is missing."""
fake_report_date = self.fake.date_time().replace(day=1)
out = self.report_downloader.download_report(fake_report_date)
self.assertEqual(out, [])
@patch("masu.external.report_downloader.ReportStatsDBAccessor")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSessionDownloadError)
def test_download_report_missing_bucket(self, mock_stats, fake_session):
"""Test download fails when bucket is missing."""
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime("%Y%m%dT000000.000Z")
expected_assembly_id = "882083b7-ea62-4aab-aa6a-f0d08d65ee2b"
input_key = f"/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz"
mock_manifest = {
"assemblyId": expected_assembly_id,
"billingPeriod": {"start": fake_report_date_str},
"reportKeys": [input_key],
}
with patch.object(AWSReportDownloader, "_get_manifest", return_value=("", mock_manifest)):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type=Provider.PROVIDER_AWS,
provider_uuid=self.aws_provider_uuid,
)
AWSReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.auth_credential,
"bucket": self.fake_bucket_name,
"report_name": self.fake_report_name,
"provider_uuid": self.aws_provider_uuid,
}
)
report_downloader.download_report(fake_report_date)
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_missing_report_name(self, fake_session):
"""Test downloading a report with an invalid report name."""
auth_credential = fake_arn(service="iam", generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.mock_task, self.fake_customer_name, auth_credential, "s3_bucket", "wrongreport")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_download_default_report(self, fake_session):
"""Test assume aws role works."""
# actual test
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
self.assertEqual(downloader.report_name, self.fake_report_name)
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSessionNoReport)
@patch("masu.util.aws.common.get_cur_report_definitions", return_value=[])
def test_download_default_report_no_report_found(self, fake_session, fake_report_list):
"""Test download fails when no reports are found."""
auth_credential = fake_arn(service="iam", generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name)
@patch("masu.external.downloader.aws.aws_report_downloader.shutil")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_check_size_success(self, fake_session, fake_shutil):
"""Test _check_size is successful."""
fake_client = Mock()
fake_client.get_object.return_value = {"ContentLength": 123456, "Body": Mock()}
fake_shutil.disk_usage.return_value = (10, 10, 4096 * 1024 * 1024)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension=random.choice(["json", "csv.gz"]))
result = downloader._check_size(fakekey, check_inflate=False)
self.assertTrue(result)
@patch("masu.external.downloader.aws.aws_report_downloader.shutil")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_check_size_fail_nospace(self, fake_session, fake_shutil):
"""Test _check_size fails if there is no more space."""
fake_client = Mock()
fake_client.get_object.return_value = {"ContentLength": 123456, "Body": Mock()}
fake_shutil.disk_usage.return_value = (10, 10, 10)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension=random.choice(["json", "csv.gz"]))
result = downloader._check_size(fakekey, check_inflate=False)
self.assertFalse(result)
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_check_size_fail_nosize(self, fake_session):
"""Test _check_size fails if there report has no size."""
fake_client = Mock()
fake_client.get_object.return_value = {}
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension=random.choice(["json", "csv.gz"]))
with self.assertRaises(AWSReportDownloaderError):
downloader._check_size(fakekey, check_inflate=False)
@patch("masu.external.downloader.aws.aws_report_downloader.shutil")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_check_size_inflate_success(self, fake_session, fake_shutil):
"""Test _check_size inflation succeeds."""
fake_client = Mock()
fake_client.get_object.return_value = {"ContentLength": 123456, "Body": io.BytesIO(b"\xd2\x02\x96I")}
fake_shutil.disk_usage.return_value = (10, 10, 4096 * 1024 * 1024)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension="csv.gz")
result = downloader._check_size(fakekey, check_inflate=True)
self.assertTrue(result)
@patch("masu.external.downloader.aws.aws_report_downloader.shutil")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_check_size_inflate_fail(self, fake_session, fake_shutil):
"""Test _check_size fails when inflation fails."""
fake_client = Mock()
fake_client.get_object.return_value = {"ContentLength": 123456, "Body": io.BytesIO(b"\xd2\x02\x96I")}
fake_shutil.disk_usage.return_value = (10, 10, 1234567)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension="csv.gz")
result = downloader._check_size(fakekey, check_inflate=True)
self.assertFalse(result)
@patch("masu.external.downloader.aws.aws_report_downloader.shutil")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_download_file_check_size_fail(self, fake_session, fake_shutil):
"""Test _check_size fails when key is fake."""
fake_client = Mock()
fake_client.get_object.return_value = {"ContentLength": 123456, "Body": io.BytesIO(b"\xd2\x02\x96I")}
fake_shutil.disk_usage.return_value = (10, 10, 1234567)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension="csv.gz")
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(fakekey)
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_download_file_raise_downloader_err(self, fake_session):
"""Test _check_size fails when there is a downloader error."""
fake_response = {"Error": {"Code": self.fake.word()}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(fake_response, "masu-test")
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(self.fake.file_path())
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_download_file_raise_nofile_err(self, fake_session):
"""Test that downloading a nonexistent file fails with AWSReportDownloaderNoFileError."""
fake_response = {"Error": {"Code": "NoSuchKey"}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(fake_response, "masu-test")
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderNoFileError):
downloader.download_file(self.fake.file_path())
@patch(
"masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.check_if_manifest_should_be_downloaded"
)
@patch("masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._remove_manifest_file")
@patch("masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._get_manifest")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_get_report_context_for_date_should_download(self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that data is returned on the reports to process."""
current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name,
provider_uuid=self.aws_provider_uuid,
)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = downloader.report.get("Compression")
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {"start": start_str},
},
)
mock_check.return_value = True
expected = {"manifest_id": None, "assembly_id": assembly_id, "compression": compression, "files": report_keys}
result = downloader.get_report_context_for_date(current_month)
with ReportManifestDBAccessor() as manifest_accessor:
manifest_entry = manifest_accessor.get_manifest(assembly_id, self.aws_provider_uuid)
expected["manifest_id"] = manifest_entry.id
self.assertIsInstance(result, dict)
for key, value in result.items():
self.assertIn(key, expected)
self.assertEqual(value, expected.get(key))
@patch(
"masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.check_if_manifest_should_be_downloaded"
)
@patch("masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._remove_manifest_file")
@patch("masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._get_manifest")
@patch("masu.util.aws.common.get_assume_role_session", return_value=FakeSession)
def test_get_report_context_for_date_should_not_download(
self, mock_session, mock_manifest, mock_delete, mock_check
):
"""Test that no data is returned when we don't want to process."""
current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = downloader.report.get("Compression")
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {"start": start_str},
},
)
mock_check.return_value = False
expected = {}
result = downloader.get_report_context_for_date(current_month)
self.assertEqual(result, expected)
def test_remove_manifest_file(self):
"""Test that we remove the manifest file."""
manifest_file = f"{DATA_DIR}/test_manifest.json"
with open(manifest_file, "w") as f:
f.write("Test")
self.assertTrue(os.path.isfile(manifest_file))
self.aws_report_downloader._remove_manifest_file(manifest_file)
self.assertFalse(os.path.isfile(manifest_file))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.aws.aws_report_downloader", level="INFO"
) as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.aws_report_downloader._remove_manifest_file("None")
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg="The log is expected to start with 'INFO:' but instead was: " + captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
class AWSLocalReportDownloaderTest(MasuTestCase):
"""Test Cases for the Local Report Downloader."""
fake = Faker()
@classmethod
def setUpClass(cls):
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = 'koku-local'
cls.fake_bucket_prefix = PREFIX
cls.selected_region = REGION
cls.fake_auth_credential = fake_arn(service='iam', generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
@classmethod
def tearDownClass(cls):
cls.manifest_accessor.close_session()
def setUp(self):
"""Set up each test."""
self.fake_bucket_name = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket.tar.gz')
mytar.extractall(path=self.fake_bucket_name)
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(
self.fake_customer_name,
self.fake_auth_credential,
self.fake_bucket_name,
'AWS-local',
1,
)
self.aws_local_report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': self.fake_bucket_name,
'provider_id': 1,
}
)
def tearDown(self):
shutil.rmtree(DATA_DIR, ignore_errors=True)
shutil.rmtree(self.fake_bucket_name)
manifests = self.manifest_accessor._get_db_obj_query().all()
for manifest in manifests:
self.manifest_accessor.delete(manifest)
self.manifest_accessor.commit()
def test_download_bucket(self):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, 'today', return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(
DATA_DIR, self.fake_customer_name, 'aws-local'
)
self.assertTrue(os.path.isdir(expected_path))
def test_report_name_provided(self):
"""Test initializer when report_name is provided."""
report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': self.fake_bucket_name,
'report_name': 'awesome-report',
}
)
self.assertEqual(report_downloader.report_name, 'awesome-report')
def test_extract_names_no_prefix(self):
"""Test to extract the report and prefix names from a bucket with no prefix."""
report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': self.fake_bucket_name,
}
)
self.assertEqual(report_downloader.report_name, self.fake_report_name)
self.assertIsNone(report_downloader.report_prefix)
def test_download_bucket_with_prefix(self):
"""Test to verify that basic report downloading works."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz')
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=8, day=7)
with patch.object(DateAccessor, 'today', return_value=test_report_date):
report_downloader = ReportDownloader(
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
'AWS-local',
1,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(
DATA_DIR, self.fake_customer_name, 'aws-local'
)
self.assertTrue(os.path.isdir(expected_path))
shutil.rmtree(fake_bucket)
def test_extract_names_with_prefix(self):
"""Test to extract the report and prefix names from a bucket with prefix."""
bucket = tempfile.mkdtemp()
report_name = 'report-name'
prefix_name = 'prefix-name'
full_path = '{}/{}/{}/20180801-20180901/'.format(
bucket, prefix_name, report_name
)
os.makedirs(full_path)
report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': bucket,
}
)
self.assertEqual(report_downloader.report_name, report_name)
self.assertEqual(report_downloader.report_prefix, prefix_name)
shutil.rmtree(full_path)
def test_extract_names_with_bad_path(self):
"""Test to extract the report and prefix names from a bad path."""
bucket = tempfile.mkdtemp()
report_name = 'report-name'
prefix_name = 'prefix-name'
full_path = '{}/{}/{}/20180801-aaaaaaa/'.format(
bucket, prefix_name, report_name
)
os.makedirs(full_path)
report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': bucket,
}
)
self.assertIsNone(report_downloader.report_name)
self.assertIsNone(report_downloader.report_prefix)
shutil.rmtree(full_path)
def test_extract_names_with_incomplete_path(self):
"""Test to extract the report and prefix from a path where a CUR hasn't been generated yet."""
bucket = tempfile.mkdtemp()
report_downloader = AWSLocalReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.fake_auth_credential,
'bucket': bucket,
}
)
self.assertIsNone(report_downloader.report_name)
self.assertIsNone(report_downloader.report_prefix)
shutil.rmtree(bucket)
def test_download_missing_month(self):
"""Test to verify that downloading a non-existant month throws proper exception."""
fake_bucket = tempfile.mkdtemp()
mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz')
mytar.extractall(fake_bucket)
test_report_date = datetime(year=2018, month=7, day=7)
with patch.object(DateAccessor, 'today', return_value=test_report_date):
report_downloader = ReportDownloader(
self.fake_customer_name,
self.fake_auth_credential,
fake_bucket,
'AWS-local',
1,
)
# Names from test report .gz file
report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(
DATA_DIR, self.fake_customer_name, 'aws-local'
)
self.assertFalse(os.path.isdir(expected_path))
class AWSReportDownloaderTest(MasuTestCase):
"""Test Cases for the AWS S3 functions."""
fake = Faker()
@classmethod
def setUpClass(cls):
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = REPORT
cls.fake_bucket_prefix = PREFIX
cls.fake_bucket_name = BUCKET
cls.selected_region = REGION
cls.auth_credential = fake_arn(service='iam', generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
@classmethod
def tearDownClass(cls):
cls.manifest_accessor.close_session()
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def setUp(self, fake_session):
os.makedirs(DATA_DIR, exist_ok=True)
self.report_downloader = ReportDownloader(self.fake_customer_name,
self.auth_credential,
self.fake_bucket_name, 'AWS',
1)
self.aws_report_downloader = AWSReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_id': 1
})
def tearDown(self):
shutil.rmtree(DATA_DIR, ignore_errors=True)
manifests = self.manifest_accessor._get_db_obj_query().all()
for manifest in manifests:
self.manifest_accessor.delete(manifest)
self.manifest_accessor.commit()
@patch('masu.external.downloader.aws.aws_report_downloader.boto3.resource')
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.download_file',
return_value=(
'mock_file_name',
None,
))
def test_download_bucket(self, mock_boto_resource, mock_download_file):
"""Test download bucket method."""
mock_resource = Mock()
mock_bucket = Mock()
mock_bucket.objects.all.return_value = []
mock_resource.Bucket.return_value = mock_bucket
mock_boto_resource = mock_resource
out = self.aws_report_downloader.download_bucket()
expected_files = []
self.assertEqual(out, expected_files)
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.download_file',
side_effect=mock_download_file_error)
def test_download_report_missing_manifest(self, mock_download_file):
fake_report_date = self.fake.date_time().replace(day=1)
out = self.report_downloader.download_report(fake_report_date)
self.assertEqual(out, [])
@patch('masu.external.report_downloader.ReportStatsDBAccessor')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSessionDownloadError)
def test_download_report_missing_bucket(self, mock_stats, fake_session):
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime('%Y%m%dT000000.000Z')
expected_assembly_id = '882083b7-ea62-4aab-aa6a-f0d08d65ee2b'
input_key = f'/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz'
mock_manifest = {
'assemblyId': expected_assembly_id,
'billingPeriod': {
'start': fake_report_date_str
},
'reportKeys': [input_key]
}
with patch.object(AWSReportDownloader,
'_get_manifest',
return_value=mock_manifest):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(self.fake_customer_name,
self.auth_credential,
self.fake_bucket_name,
'AWS', 2)
AWSReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_id': 2
})
report_downloader.download_report(fake_report_date)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_missing_report_name(self, fake_session):
"""Test downloading a report with an invalid report name."""
auth_credential = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.fake_customer_name, auth_credential,
's3_bucket', 'wrongreport')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_default_report(self, fake_session):
# actual test
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
self.assertEqual(downloader.report_name, self.fake_report_name)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSessionNoReport)
@patch('masu.util.aws.common.get_cur_report_definitions', return_value=[])
def test_download_default_report_no_report_found(self, fake_session,
fake_report_list):
auth_credential = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.fake_customer_name, auth_credential,
self.fake_bucket_name)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_success(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': Mock()
}
fake_shutil.disk_usage.return_value = (10, 10, 4096 * 1024 * 1024)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension=random.choice(
['json', 'csv.gz']))
result = downloader._check_size(fakekey, check_inflate=False)
self.assertTrue(result)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_fail_nospace(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': Mock()
}
fake_shutil.disk_usage.return_value = (10, 10, 10)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension=random.choice(
['json', 'csv.gz']))
result = downloader._check_size(fakekey, check_inflate=False)
self.assertFalse(result)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_fail_nosize(self, fake_session):
fake_client = Mock()
fake_client.get_object.return_value = {}
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension=random.choice(
['json', 'csv.gz']))
with self.assertRaises(AWSReportDownloaderError):
downloader._check_size(fakekey, check_inflate=False)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_inflate_success(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': io.BytesIO(b'\xd2\x02\x96I')
}
fake_shutil.disk_usage.return_value = (10, 10, 4096 * 1024 * 1024)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension='csv.gz')
result = downloader._check_size(fakekey, check_inflate=True)
self.assertTrue(result)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_inflate_fail(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': io.BytesIO(b'\xd2\x02\x96I')
}
fake_shutil.disk_usage.return_value = (10, 10, 1234567)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension='csv.gz')
result = downloader._check_size(fakekey, check_inflate=True)
self.assertFalse(result)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_file_check_size_fail(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': io.BytesIO(b'\xd2\x02\x96I')
}
fake_shutil.disk_usage.return_value = (10, 10, 1234567)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension='csv.gz')
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(fakekey)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_file_raise_downloader_err(self, fake_session):
fake_response = {'Error': {'Code': self.fake.word()}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(
fake_response, 'masu-test')
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(self.fake.file_path())
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_file_raise_nofile_err(self, fake_session):
fake_response = {'Error': {'Code': 'NoSuchKey'}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(
fake_response, 'masu-test')
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderNoFileError):
downloader.download_file(self.fake.file_path())
class AWSReportDownloaderTest(MasuTestCase):
"""Test Cases for the AWS S3 functions."""
fake = Faker()
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = REPORT
cls.fake_bucket_prefix = PREFIX
cls.fake_bucket_name = BUCKET
cls.selected_region = REGION
cls.auth_credential = fake_arn(service='iam', generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def setUp(self, fake_session):
super().setUp()
os.makedirs(DATA_DIR, exist_ok=True)
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type='AWS',
provider_uuid=self.aws_provider_uuid,
)
self.aws_report_downloader = AWSReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_uuid': self.aws_provider_uuid,
})
def tearDown(self):
shutil.rmtree(DATA_DIR, ignore_errors=True)
@patch('masu.external.downloader.aws.aws_report_downloader.boto3.resource')
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.download_file',
return_value=('mock_file_name', None),
)
def test_download_bucket(self, mock_boto_resource, mock_download_file):
"""Test download bucket method."""
mock_resource = Mock()
mock_bucket = Mock()
mock_bucket.objects.all.return_value = []
mock_resource.Bucket.return_value = mock_bucket
mock_boto_resource = mock_resource
out = self.aws_report_downloader.download_bucket()
expected_files = []
self.assertEqual(out, expected_files)
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.download_file',
side_effect=mock_download_file_error,
)
def test_download_report_missing_manifest(self, mock_download_file):
fake_report_date = self.fake.date_time().replace(day=1)
out = self.report_downloader.download_report(fake_report_date)
self.assertEqual(out, [])
@patch('masu.external.report_downloader.ReportStatsDBAccessor')
@patch(
'masu.util.aws.common.get_assume_role_session',
return_value=FakeSessionDownloadError,
)
def test_download_report_missing_bucket(self, mock_stats, fake_session):
mock_stats.return_value.__enter__ = Mock()
fake_report_date = self.fake.date_time().replace(day=1)
fake_report_date_str = fake_report_date.strftime('%Y%m%dT000000.000Z')
expected_assembly_id = '882083b7-ea62-4aab-aa6a-f0d08d65ee2b'
input_key = f'/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz'
mock_manifest = {
'assemblyId': expected_assembly_id,
'billingPeriod': {
'start': fake_report_date_str
},
'reportKeys': [input_key],
}
with patch.object(AWSReportDownloader,
'_get_manifest',
return_value=('', mock_manifest)):
with self.assertRaises(AWSReportDownloaderError):
report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.auth_credential,
report_source=self.fake_bucket_name,
provider_type='AWS',
provider_uuid=self.aws_provider_uuid,
)
AWSReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.auth_credential,
'bucket': self.fake_bucket_name,
'report_name': self.fake_report_name,
'provider_uuid': self.aws_provider_uuid,
})
report_downloader.download_report(fake_report_date)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_missing_report_name(self, fake_session):
"""Test downloading a report with an invalid report name."""
auth_credential = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.mock_task, self.fake_customer_name,
auth_credential, 's3_bucket', 'wrongreport')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_default_report(self, fake_session):
# actual test
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
self.assertEqual(downloader.report_name, self.fake_report_name)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSessionNoReport)
@patch('masu.util.aws.common.get_cur_report_definitions', return_value=[])
def test_download_default_report_no_report_found(self, fake_session,
fake_report_list):
auth_credential = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.mock_task, self.fake_customer_name,
auth_credential, self.fake_bucket_name)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_success(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': Mock()
}
fake_shutil.disk_usage.return_value = (10, 10, 4096 * 1024 * 1024)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension=random.choice(
['json', 'csv.gz']))
result = downloader._check_size(fakekey, check_inflate=False)
self.assertTrue(result)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_fail_nospace(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': Mock()
}
fake_shutil.disk_usage.return_value = (10, 10, 10)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension=random.choice(
['json', 'csv.gz']))
result = downloader._check_size(fakekey, check_inflate=False)
self.assertFalse(result)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_fail_nosize(self, fake_session):
fake_client = Mock()
fake_client.get_object.return_value = {}
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension=random.choice(
['json', 'csv.gz']))
with self.assertRaises(AWSReportDownloaderError):
downloader._check_size(fakekey, check_inflate=False)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_inflate_success(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': io.BytesIO(b'\xd2\x02\x96I'),
}
fake_shutil.disk_usage.return_value = (10, 10, 4096 * 1024 * 1024)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension='csv.gz')
result = downloader._check_size(fakekey, check_inflate=True)
self.assertTrue(result)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_check_size_inflate_fail(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': io.BytesIO(b'\xd2\x02\x96I'),
}
fake_shutil.disk_usage.return_value = (10, 10, 1234567)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension='csv.gz')
result = downloader._check_size(fakekey, check_inflate=True)
self.assertFalse(result)
@patch('masu.external.downloader.aws.aws_report_downloader.shutil')
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_file_check_size_fail(self, fake_session, fake_shutil):
fake_client = Mock()
fake_client.get_object.return_value = {
'ContentLength': 123456,
'Body': io.BytesIO(b'\xd2\x02\x96I'),
}
fake_shutil.disk_usage.return_value = (10, 10, 1234567)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5),
extension='csv.gz')
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(fakekey)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_file_raise_downloader_err(self, fake_session):
fake_response = {'Error': {'Code': self.fake.word()}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(
fake_response, 'masu-test')
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(self.fake.file_path())
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_download_file_raise_nofile_err(self, fake_session):
fake_response = {'Error': {'Code': 'NoSuchKey'}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(
fake_response, 'masu-test')
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderNoFileError):
downloader.download_file(self.fake.file_path())
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.check_if_manifest_should_be_downloaded'
)
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._remove_manifest_file'
)
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._get_manifest'
)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_get_report_context_for_date_should_download(
self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that data is returned on the reports to process."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name,
provider_uuid=self.aws_provider_uuid)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = '1234'
compression = downloader.report.get('Compression')
report_keys = ['file1', 'file2']
mock_manifest.return_value = ('', {
'assemblyId': assembly_id,
'Compression': compression,
'reportKeys': report_keys,
'billingPeriod': {
'start': start_str
}
})
mock_check.return_value = True
expected = {
'manifest_id': None,
'assembly_id': assembly_id,
'compression': compression,
'files': report_keys
}
result = downloader.get_report_context_for_date(current_month)
with ReportManifestDBAccessor() as manifest_accessor:
manifest_entry = manifest_accessor.get_manifest(
assembly_id, self.aws_provider_uuid)
expected['manifest_id'] = manifest_entry.id
self.assertIsInstance(result, dict)
for key, value in result.items():
self.assertIn(key, expected)
self.assertEqual(value, expected.get(key))
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader.check_if_manifest_should_be_downloaded'
)
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._remove_manifest_file'
)
@patch(
'masu.external.downloader.aws.aws_report_downloader.AWSReportDownloader._get_manifest'
)
@patch('masu.util.aws.common.get_assume_role_session',
return_value=FakeSession)
def test_get_report_context_for_date_should_not_download(
self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that no data is returned when we don't want to process."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = '1234'
compression = downloader.report.get('Compression')
report_keys = ['file1', 'file2']
mock_manifest.return_value = ('', {
'assemblyId': assembly_id,
'Compression': compression,
'reportKeys': report_keys,
'billingPeriod': {
'start': start_str
}
})
mock_check.return_value = False
expected = {}
result = downloader.get_report_context_for_date(current_month)
self.assertEqual(result, expected)
def test_remove_manifest_file(self):
manifest_file = f'{DATA_DIR}/test_manifest.json'
with open(manifest_file, 'w') as f:
f.write('Test')
self.assertTrue(os.path.isfile(manifest_file))
self.aws_report_downloader._remove_manifest_file(manifest_file)
self.assertFalse(os.path.isfile(manifest_file))
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
self.credentials = {"cluster_id": self.cluster_id}
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source={},
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {},
"provider_uuid": self.ocp_provider_uuid,
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
@patch("masu.util.aws.common.copy_data_to_s3_bucket", return_value=None)
def test_download_bucket(self, mock_copys3):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def test_divide_csv_daily(self):
"""Test the divide_csv_daily method."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_report = {
"interval_start": [
"2020-01-01 00:00:00 +UTC",
"2020-01-02 00:00:00 +UTC"
],
"persistentvolumeclaim_labels": ["label1", "label2"],
}
df = pd.DataFrame(data=mock_report)
mock_pd.read_csv.return_value = df
daily_files = divide_csv_daily(file_path, filename)
self.assertNotEqual([], daily_files)
self.assertEqual(len(daily_files), 2)
gen_files = [
"storage_usage.2020-01-01.csv",
"storage_usage.2020-01-02.csv"
]
expected = [{
"filename": gen_file,
"filepath": f"{td}/{gen_file}"
} for gen_file in gen_files]
for expected_item in expected:
self.assertIn(expected_item, daily_files)
def test_divide_csv_daily_failure(self):
"""Test the divide_csv_daily method throw error on reading CSV."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
errorMsg = "CParserError: Error tokenizing data. C error: Expected 53 fields in line 1605634, saw 54"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_pd.read_csv.side_effect = Exception(errorMsg)
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.LOG.error"
) as mock_debug:
with self.assertRaises(Exception):
divide_csv_daily(file_path, filename)
mock_debug.assert_called_once_with(
f"File {file_path} could not be parsed. Reason: {errorMsg}"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._get_manifest"
)
def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
}
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
@override_settings(ENABLE_S3_ARCHIVING=True)
@override_settings(ENABLE_PARQUET_PROCESSING=True)
@patch("masu.external.downloader.ocp.ocp_report_downloader.os")
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.copy_local_report_file_to_s3_bucket"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.divide_csv_daily")
def test_create_daily_archives(self, mock_divide, mock_s3_copy, mock_os):
"""Test that this method returns a file list."""
start_date = DateHelper().this_month_start
daily_files = [
{
"filename": "file_one",
"filepath": "path/to/file_one"
},
{
"filename": "file_two",
"filepath": "path/to/file_two"
},
]
expected_filenames = ["path/to/file_one", "path/to/file_two"]
mock_divide.return_value = daily_files
result = create_daily_archives(1, "10001", self.ocp_provider_uuid,
"file", "path", 1, start_date)
self.assertEqual(result, expected_filenames)
def _get_report_files(
tracing_id,
customer_name,
authentication,
billing_source,
provider_type,
provider_uuid,
report_month,
report_context,
):
"""
Task to download a Report.
Args:
tracing_id (String): Tracing ID for file processing.
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
provider_uuid (String): Provider uuid.
report_month (DateTime): Month for report to download.
cache_key (String): The provider specific task cache value.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
context = {"account": customer_name[4:], "provider_uuid": provider_uuid}
month_string = report_month.strftime("%B %Y")
report_context["date"] = report_month
function_name = "masu.processor._tasks.download._get_report_files"
log_statement = (
f"{function_name}: "
f"Downloading report for: "
f" schema_name: {customer_name} "
f" provider: {provider_type} "
f" account (provider uuid): {provider_uuid} "
f" report_month: {month_string} "
)
LOG.info(log_json(tracing_id, log_statement, context))
try:
disk = psutil.disk_usage(Config.PVC_DIR)
disk_msg = f"{function_name}: Available disk space: {disk.free} bytes ({100 - disk.percent}%)"
except OSError:
disk_msg = f"{function_name}: Unable to find available disk space. {Config.PVC_DIR} does not exist"
LOG.info(log_json(tracing_id, disk_msg, context))
downloader = ReportDownloader(
customer_name=customer_name,
credentials=authentication,
data_source=billing_source,
provider_type=provider_type,
provider_uuid=provider_uuid,
report_name=None,
account=customer_name[4:],
tracing_id=tracing_id,
)
return downloader.download_report(report_context)
