class AWSLocalReportDownloaderTest(MasuTestCase): """Test Cases for the Local Report Downloader.""" fake = Faker() @classmethod def setUpClass(cls): """Set up class variables.""" super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = "koku-local" cls.fake_bucket_prefix = PREFIX cls.selected_region = REGION cls.fake_auth_credential = fake_arn(service="iam", generate_account_id=True) cls.manifest_accessor = ReportManifestDBAccessor() def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz") mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.fake_auth_credential, report_source=self.fake_bucket_name, provider_type=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": self.fake_bucket_name, "provider_uuid": self.aws_provider_uuid, } ) def tearDown(self): """Remove test generated data.""" shutil.rmtree(DATA_DIR, ignore_errors=True) shutil.rmtree(self.fake_bucket_name) def test_download_bucket(self): """Test to verify that basic report downloading works.""" test_report_date = datetime(year=2018, month=8, day=7) with patch.object(DateAccessor, "today", return_value=test_report_date): self.report_downloader.download_report(test_report_date) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertTrue(os.path.isdir(expected_path)) def test_report_name_provided(self): """Test initializer when report_name is provided.""" report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": self.fake_bucket_name, "report_name": "awesome-report", } ) self.assertEqual(report_downloader.report_name, "awesome-report") def test_extract_names_no_prefix(self): """Test to extract the report and prefix names from a bucket with no prefix.""" report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": self.fake_bucket_name, } ) self.assertEqual(report_downloader.report_name, self.fake_report_name) self.assertIsNone(report_downloader.report_prefix) def test_download_bucket_with_prefix(self): """Test to verify that basic report downloading works.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz") mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=8, day=7) with patch.object(DateAccessor, "today", return_value=test_report_date): report_downloader = ReportDownloader( self.mock_task, self.fake_customer_name, self.fake_auth_credential, fake_bucket, Provider.PROVIDER_AWS_LOCAL, self.aws_provider_uuid, ) # Names from test report .gz file report_downloader.download_report(test_report_date) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertTrue(os.path.isdir(expected_path)) shutil.rmtree(fake_bucket) def test_extract_names_with_prefix(self): """Test to extract the report and prefix names from a bucket with prefix.""" bucket = tempfile.mkdtemp() report_name = "report-name" prefix_name = "prefix-name" full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-20180901/" os.makedirs(full_path) report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": bucket, } ) self.assertEqual(report_downloader.report_name, report_name) self.assertEqual(report_downloader.report_prefix, prefix_name) shutil.rmtree(full_path) def test_extract_names_with_bad_path(self): """Test to extract the report and prefix names from a bad path.""" bucket = tempfile.mkdtemp() report_name = "report-name" prefix_name = "prefix-name" full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-aaaaaaa/" os.makedirs(full_path) report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": bucket, } ) self.assertIsNone(report_downloader.report_name) self.assertIsNone(report_downloader.report_prefix) shutil.rmtree(full_path) def test_extract_names_with_incomplete_path(self): """Test to extract the report and prefix from a path where a CUR hasn't been generated yet.""" bucket = tempfile.mkdtemp() report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": bucket, } ) self.assertIsNone(report_downloader.report_name) self.assertIsNone(report_downloader.report_prefix) shutil.rmtree(bucket) def test_download_missing_month(self): """Test to verify that downloading a non-existant month throws proper exception.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz") mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=7, day=7) with patch.object(DateAccessor, "today", return_value=test_report_date): report_downloader = ReportDownloader( self.mock_task, self.fake_customer_name, self.fake_auth_credential, fake_bucket, Provider.PROVIDER_AWS_LOCAL, 1, ) # Names from test report .gz file report_downloader.download_report(test_report_date) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertFalse(os.path.isdir(expected_path)) def test_delete_manifest_file_warning(self): """Test that an INFO is logged when removing a manifest file that does not exist.""" with self.assertLogs( logger="masu.external.downloader.aws_local.aws_local_report_downloader", level="INFO" ) as captured_logs: # Disable log suppression logging.disable(logging.NOTSET) self.aws_local_report_downloader._remove_manifest_file("None") self.assertTrue( captured_logs.output[0].startswith("INFO:"), msg="The log is expected to start with 'INFO:' but instead was: " + captured_logs.output[0], ) self.assertTrue( "Could not delete manifest file at" in captured_logs.output[0], msg="""The log message is expected to contain 'Could not delete manifest file at' but instead was: """ + captured_logs.output[0], ) # Re-enable log suppression logging.disable(logging.CRITICAL)
class AWSLocalReportDownloaderTest(MasuTestCase): """Test Cases for the Local Report Downloader.""" fake = Faker() @classmethod def setUpClass(cls): """Set up class variables.""" super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = "koku-local" cls.fake_bucket_prefix = PREFIX cls.selected_region = REGION cls.fake_auth_credential = fake_arn(service="iam", generate_account_id=True) cls.manifest_accessor = ReportManifestDBAccessor() def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz") mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.credentials = {"role_arn": self.fake_auth_credential} self.data_source = {"bucket": self.fake_bucket_name} self.report_downloader = ReportDownloader( customer_name=self.fake_customer_name, credentials=self.credentials, data_source=self.data_source, provider_type=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": self.data_source, "provider_uuid": self.aws_provider_uuid, }) def tearDown(self): """Remove test generated data.""" shutil.rmtree(DATA_DIR, ignore_errors=True) shutil.rmtree(self.fake_bucket_name) @patch("masu.processor.parquet.parquet_report_processor.settings", ENABLE_S3_ARCHIVING=True) def test_download_bucket(self, _): """Test to verify that basic report downloading works.""" with patch("masu.processor.parquet.parquet_report_processor.Path"): with patch("masu.processor.parquet.parquet_report_processor.pd"): with patch( "masu.processor.parquet.parquet_report_processor.open" ): with patch( "masu.processor.parquet.parquet_report_processor.copy_data_to_s3_bucket" ): with patch( "masu.processor.parquet.parquet_report_processor.ParquetReportProcessor." "create_parquet_table"): test_report_date = datetime(year=2018, month=8, day=7) with patch.object(DateAccessor, "today", return_value=test_report_date): report_context = { "date": test_report_date.date(), "manifest_id": 1, "comporession": "GZIP", "current_file": "./koku/masu/test/data/test_local_bucket.tar.gz", } self.report_downloader.download_report( report_context) expected_path = "{}/{}/{}".format( DATA_DIR, self.fake_customer_name, "aws-local") self.assertTrue(os.path.isdir(expected_path)) def test_report_name_provided(self): """Test initializer when report_name is provided.""" report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": self.data_source, "report_name": "awesome-report", }) self.assertEqual(report_downloader.report_name, "awesome-report") @patch("masu.processor.parquet.parquet_report_processor.settings", ENABLE_S3_ARCHIVING=True) def test_extract_names_no_prefix(self, _): """Test to extract the report and prefix names from a bucket with no prefix.""" with patch("masu.processor.parquet.parquet_report_processor.Path"): with patch("masu.processor.parquet.parquet_report_processor.pd"): with patch( "masu.processor.parquet.parquet_report_processor.open" ): with patch( "masu.processor.parquet.parquet_report_processor.copy_data_to_s3_bucket" ): with patch( "masu.processor.parquet.parquet_report_processor.ParquetReportProcessor." "create_parquet_table"): report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": self.data_source, }) self.assertEqual(report_downloader.report_name, self.fake_report_name) self.assertIsNone(report_downloader.report_prefix) def test_download_bucket_with_prefix(self): """Test to verify that basic report downloading works.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open( "./koku/masu/test/data/test_local_bucket_prefix.tar.gz") mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=8, day=7) fake_data_source = {"bucket": fake_bucket} with patch.object(DateAccessor, "today", return_value=test_report_date): report_downloader = ReportDownloader( self.fake_customer_name, self.credentials, fake_data_source, Provider.PROVIDER_AWS_LOCAL, self.aws_provider_uuid, ) # Names from test report .gz file report_context = { "date": test_report_date.date(), "manifest_id": 1, "comporession": "GZIP", "current_file": "./koku/masu/test/data/test_local_bucket.tar.gz", } report_downloader.download_report(report_context) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertTrue(os.path.isdir(expected_path)) shutil.rmtree(fake_bucket) def test_extract_names_with_prefix(self): """Test to extract the report and prefix names from a bucket with prefix.""" bucket = tempfile.mkdtemp() report_name = "report-name" prefix_name = "prefix-name" full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-20180901/" os.makedirs(full_path) report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": { "bucket": bucket }, }) self.assertEqual(report_downloader.report_name, report_name) self.assertEqual(report_downloader.report_prefix, prefix_name) shutil.rmtree(full_path) def test_extract_names_with_bad_path(self): """Test to extract the report and prefix names from a bad path.""" bucket = tempfile.mkdtemp() report_name = "report-name" prefix_name = "prefix-name" full_path = f"{bucket}/{prefix_name}/{report_name}/20180801-aaaaaaa/" os.makedirs(full_path) report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": { "bucket": bucket }, }) self.assertIsNone(report_downloader.report_name) self.assertIsNone(report_downloader.report_prefix) shutil.rmtree(full_path) def test_extract_names_with_incomplete_path(self): """Test to extract the report and prefix from a path where a CUR hasn't been generated yet.""" bucket = tempfile.mkdtemp() report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": { "bucket": bucket }, }) self.assertIsNone(report_downloader.report_name) self.assertIsNone(report_downloader.report_prefix) shutil.rmtree(bucket) def test_delete_manifest_file_warning(self): """Test that an INFO is logged when removing a manifest file that does not exist.""" with self.assertLogs( logger= "masu.external.downloader.aws_local.aws_local_report_downloader", level="INFO") as captured_logs: # Disable log suppression logging.disable(logging.NOTSET) self.aws_local_report_downloader._remove_manifest_file("None") self.assertTrue( captured_logs.output[0].startswith("INFO:"), msg= "The log is expected to start with 'INFO:' but instead was: " + captured_logs.output[0], ) self.assertTrue( "Could not delete manifest file at" in captured_logs.output[0], msg="""The log message is expected to contain 'Could not delete manifest file at' but instead was: """ + captured_logs.output[0], ) # Re-enable log suppression logging.disable(logging.CRITICAL) @patch( "masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._remove_manifest_file" ) @patch( "masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._get_manifest" ) def test_get_manifest_context_for_date(self, mock_manifest, mock_delete): """Test that the manifest is read.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) downloader = AWSLocalReportDownloader( self.fake_customer_name, self.credentials, self.data_source, provider_uuid=self.aws_provider_uuid) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = "1234" compression = "GZIP" report_keys = ["file1", "file2"] mock_manifest.return_value = ( "", { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": { "start": start_str }, }, DateAccessor().today(), ) result = downloader.get_manifest_context_for_date(current_month) self.assertEqual(result.get("assembly_id"), assembly_id) self.assertEqual(result.get("compression"), compression) self.assertIsNotNone(result.get("files")) @patch( "masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._remove_manifest_file" ) @patch( "masu.external.downloader.aws_local.aws_local_report_downloader.AWSLocalReportDownloader._get_manifest" ) def test_get_manifest_context_for_date_no_manifest(self, mock_manifest, mock_delete): """Test that the manifest is read.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) downloader = AWSLocalReportDownloader( self.fake_customer_name, self.credentials, self.data_source, provider_uuid=self.aws_provider_uuid) mock_manifest.return_value = ("", { "reportKeys": [] }, DateAccessor().today()) result = downloader.get_manifest_context_for_date(current_month) self.assertEqual(result, {})