def test_get_manifest_context_for_date(self, mock_manifest, mock_delete): """Test that the manifest is read.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSLocalReportDownloader( self.fake_customer_name, auth_credential, self.fake_bucket_name, provider_uuid=self.aws_provider_uuid) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = "1234" compression = "GZIP" report_keys = ["file1", "file2"] mock_manifest.return_value = ( "", { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": { "start": start_str }, }, ) result = downloader.get_manifest_context_for_date(current_month) self.assertEqual(result.get("assembly_id"), assembly_id) self.assertEqual(result.get("compression"), compression) self.assertIsNotNone(result.get("files"))
def test_get_report_context_for_date_should_not_download( self, mock_session, mock_manifest, mock_delete, mock_check ): """Test that no data is returned when we don't want to process.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = "1234" compression = downloader.report.get("Compression") report_keys = ["file1", "file2"] mock_manifest.return_value = ( "", { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": {"start": start_str}, }, ) mock_check.return_value = False expected = {} result = downloader.get_report_context_for_date(current_month) self.assertEqual(result, expected)
def setUp(self): """Set up shared variables.""" super().setUp() self.aws_credentials = self.aws_provider.authentication.credentials self.aws_data_source = self.aws_provider.billing_source.data_source self.azure_credentials = self.azure_provider.authentication.credentials self.azure_data_source = self.azure_provider.billing_source.data_source self.gcp_credentials = self.gcp_provider.authentication.credentials self.gcp_data_source = self.gcp_provider.billing_source.data_source self.ocp_credentials = [ name[0] for name in Provider.objects.values_list( "authentication__credentials") ] self.ocp_data_source = {} self.mock_accounts = [{ "credentials": { "role_arn": fake_arn(service="iam", generate_account_id=True) }, "data_source": { "bucket": self.fake.word() }, "customer_name": self.fake.word(), "provider_type": Provider.PROVIDER_AWS, "schema_name": self.fake.word(), }]
def setUp(self): """Set up the test.""" super().setUp() self.account_id = fake_aws_account_id() self.arn = fake_arn(account_id=self.account_id, region=REGION, service="iam")
def test_download_default_report_no_report_found(self, fake_session, fake_report_list): auth_credential = fake_arn(service='iam', generate_account_id=True) with self.assertRaises(MasuProviderError): AWSReportDownloader(self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name)
def test_missing_report_name(self, fake_session): """Test downloading a report with an invalid report name.""" auth_credential = fake_arn(service='iam', generate_account_id=True) with self.assertRaises(MasuProviderError): AWSReportDownloader(self.mock_task, self.fake_customer_name, auth_credential, 's3_bucket', 'wrongreport')
def setUp(self): """Set up test case.""" super().setUp() self.schema = "acct12345" self.paginator_dict = { "r-0": { "OrganizationalUnits": [ {"Id": "ou-0", "Arn": "arn-0", "Name": "Big_Org_0"}, {"Id": "ou-1", "Arn": "arn-1", "Name": "Big_Org_1"}, {"Id": "ou-2", "Arn": "arn-2", "Name": "Big_Org_2"}, ] }, "ou-0": {"OrganizationalUnits": [{"Id": "sou-0", "Arn": "arn-0", "Name": "Sub_Org_0"}]}, "ou-1": {"OrganizationalUnits": []}, "ou-2": {"OrganizationalUnits": []}, "sou-0": {"OrganizationalUnits": []}, } self.account = { "credentials": {"role_arn": fake_arn(service="iam", generate_account_id=True)}, "customer_name": CUSTOMER_NAME, "billing_source": BUCKET, "provider_type": P_TYPE, "schema_name": self.schema, "provider_uuid": P_UUID, }
def setUp(self): """Set up shared variables.""" super().setUp() self.aws_provider_resource_name = self.aws_provider.authentication.provider_resource_name self.aws_billing_source = self.aws_provider.billing_source.bucket self.azure_credentials = self.azure_provider.authentication.credentials self.azure_data_source = self.azure_provider.billing_source.data_source self.ocp_provider_resource_names = [ name[0] for name in Provider.objects.values_list( "authentication__provider_resource_name") ] self.ocp_billing_source = None self.mock_accounts = [] self.mock_accounts.append({ "authentication": fake_arn(service="iam", generate_account_id=True), "billing_source": self.fake.word(), "customer_name": self.fake.word(), "provider_type": Provider.PROVIDER_AWS, "schema_name": self.fake.word(), })
def test_download_default_report(self, fake_session): # actual test auth_credential = fake_arn(service='iam', generate_account_id=True) downloader = AWSReportDownloader(self.fake_customer_name, auth_credential, self.fake_bucket_name) self.assertEqual(downloader.report_name, self.fake_report_name)
def setUp(self): """Set up test case.""" super().setUp() self.paginator_dict = { "r-0": { "OrganizationalUnits": [ {"Id": "ou-0", "Arn": "arn-0", "Name": "Big_Org_0"}, {"Id": "ou-1", "Arn": "arn-1", "Name": "Big_Org_1"}, {"Id": "ou-2", "Arn": "arn-2", "Name": "Big_Org_2"}, ] }, "ou-0": {"OrganizationalUnits": [{"Id": "sou-0", "Arn": "arn-0", "Name": "Sub_Org_0"}]}, "ou-1": {"OrganizationalUnits": []}, "ou-2": {"OrganizationalUnits": []}, "sou-0": {"OrganizationalUnits": []}, } self.account = { "credentials": {"role_arn": fake_arn(service="iam", generate_account_id=True)}, "customer_name": CUSTOMER_NAME, "billing_source": BUCKET, "provider_type": P_TYPE, "schema_name": self.schema, "provider_uuid": P_UUID, } with schema_context(self.schema): # Delete the rows created by the koku_test_runner. This next test suite # is intended to test how the crawler inserts the data into the database # which is easier to do without preloaded data. AWSOrganizationalUnit.objects.all().delete()
def test_parse_arn_with_region_and_account(self): """Assert successful account ID parsing from a well-formed ARN.""" mock_account_id = fake_aws_account_id() mock_arn = fake_arn(account_id=mock_account_id, region="test-region-1") arn_object = utils.AwsArn(mock_arn) partition = arn_object.partition self.assertIsNotNone(partition) service = arn_object.service self.assertIsNotNone(service) region = arn_object.region self.assertIsNotNone(region) account_id = arn_object.account_id self.assertIsNotNone(account_id) resource_type = arn_object.resource_type self.assertIsNotNone(resource_type) resource_separator = arn_object.resource_separator self.assertIsNotNone(resource_separator) resource = arn_object.resource self.assertIsNotNone(resource) reconstructed_arn = ("arn:" + partition + ":" + service + ":" + region + ":" + account_id + ":" + resource_type + resource_separator + resource) self.assertEqual(mock_account_id, account_id) self.assertEqual(mock_arn, reconstructed_arn)
def test_init_with_demo_account(self): """Test init with the demo account.""" mock_task = Mock( request=Mock(id=str(self.fake.uuid4()), return_value={})) account_id = "123456" arn = fake_arn(service="iam", generate_account_id=True) credentials = {"role_arn": arn} report_name = FAKE.word() demo_accounts = { account_id: { arn: { "report_name": report_name, "report_prefix": FAKE.word() } } } with self.settings(DEMO_ACCOUNTS=demo_accounts): with patch("masu.util.aws.common.get_assume_role_session" ) as mock_session: AWSReportDownloader( **{ "task": mock_task, "customer_name": f"acct{account_id}", "credentials": credentials, "data_source": { "bucket": FAKE.word() }, "report_name": report_name, "provider_uuid": self.aws_provider_uuid, }) mock_session.assert_called_once()
def test_get_report_context_for_date_should_not_download( self, mock_session, mock_manifest, mock_delete, mock_check): """Test that no data is returned when we don't want to process.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service='iam', generate_account_id=True) downloader = AWSReportDownloader(self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = '1234' compression = downloader.report.get('Compression') report_keys = ['file1', 'file2'] mock_manifest.return_value = ('', { 'assemblyId': assembly_id, 'Compression': compression, 'reportKeys': report_keys, 'billingPeriod': { 'start': start_str } }) mock_check.return_value = False expected = {} result = downloader.get_report_context_for_date(current_month) self.assertEqual(result, expected)
def test_disk_status_logging(self, fake_downloader): """Test task for logging when temp directory exists.""" logging.disable(logging.NOTSET) os.makedirs(Config.TMP_DIR, exist_ok=True) account = fake_arn(service="iam", generate_account_id=True) expected = "INFO:masu.processor._tasks.download:Available disk space" with self.assertLogs("masu.processor._tasks.download", level="INFO") as logger: _get_report_files( Mock(), customer_name=self.fake.word(), authentication=account, provider_type=Provider.PROVIDER_AWS, report_month=DateHelper().today, provider_uuid=self.aws_provider_uuid, billing_source=self.fake.word(), cache_key=self.fake.word(), ) statement_found = False for log in logger.output: if expected in log: statement_found = True self.assertTrue(statement_found) shutil.rmtree(Config.TMP_DIR, ignore_errors=True)
def test_download_default_report_no_report_found(self, fake_session, fake_report_list): """Test download fails when no reports are found.""" auth_credential = fake_arn(service="iam", generate_account_id=True) with self.assertRaises(MasuProviderError): AWSReportDownloader(self.fake_customer_name, auth_credential, self.fake_bucket_name)
def test_download_default_report(self, fake_session): """Test assume aws role works.""" # actual test auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) self.assertEqual(downloader.report_name, self.fake_report_name)
def setUp(self): """Set up the test.""" super().setUp() self.account_id = fake_aws_account_id() self.arn = fake_arn(account_id=self.account_id, region=REGION, service='iam') with ReportingCommonDBAccessor() as common_accessor: self.column_map = common_accessor.column_map
def setUpClass(cls): super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = REPORT cls.fake_bucket_prefix = PREFIX cls.fake_bucket_name = BUCKET cls.selected_region = REGION cls.auth_credential = fake_arn(service='iam', generate_account_id=True) cls.manifest_accessor = ReportManifestDBAccessor()
def test_parse_arn_without_region_or_account(self): """Assert successful ARN parsing without a region or an account id.""" mock_arn = fake_arn() arn_object = utils.AwsArn(mock_arn) region = arn_object.region self.assertEqual(region, None) account_id = arn_object.account_id self.assertEqual(account_id, None)
def setUpClass(cls): """Set up class variables.""" super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = "koku-local" cls.fake_bucket_prefix = PREFIX cls.selected_region = REGION cls.fake_auth_credential = fake_arn(service="iam", generate_account_id=True) cls.manifest_accessor = ReportManifestDBAccessor()
def setUpClass(cls): """Set up the class.""" super().setUpClass() cls.fake = faker.Faker() cls.fake_reports = [ {'file': cls.fake.word(), 'compression': 'GZIP'}, {'file': cls.fake.word(), 'compression': 'PLAIN'}, ] cls.fake_account = fake_arn(service='iam', generate_account_id=True) cls.today = DateAccessor().today_with_timezone('UTC') cls.yesterday = cls.today - timedelta(days=1)
def setUpClass(cls): """Set up the class.""" super().setUpClass() cls.fake = faker.Faker() cls.fake_reports = [ {"file": cls.fake.word(), "compression": "GZIP"}, {"file": cls.fake.word(), "compression": "PLAIN"}, ] cls.fake_account = fake_arn(service="iam", generate_account_id=True) cls.today = DateAccessor().today_with_timezone("UTC") cls.yesterday = cls.today - timedelta(days=1)
def test_download_file_raise_nofile_err(self, fake_session): fake_response = {'Error': {'Code': 'NoSuchKey'}} fake_client = Mock() fake_client.get_object.side_effect = ClientError(fake_response, 'masu-test') auth_credential = fake_arn(service='iam', generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) downloader.s3_client = fake_client with self.assertRaises(AWSReportDownloaderNoFileError): downloader.download_file(self.fake.file_path())
def test_get_report_exception(self, fake_downloader): """Test task""" account = fake_arn(service='iam', generate_account_id=True) with self.assertRaises(Exception): _get_report_files( customer_name=self.fake.word(), authentication=account, provider_type='AWS', report_name=self.fake.word(), provider_uuid=self.aws_test_provider_uuid, billing_source=self.fake.word(), )
def test_get_report_update_status(self, fake_downloader, fake_status): """Test that status is updated when downloading is complete.""" account = fake_arn(service='iam', generate_account_id=True) _get_report_files( customer_name=self.fake.word(), authentication=account, provider_type='AWS', report_name=self.fake.word(), provider_uuid=self.aws_test_provider_uuid, billing_source=self.fake.word(), ) fake_status.assert_called_with(ProviderStatusCode.READY)
def test_parse_arn_with_slash_separator(self): """Assert successful ARN parsing with a slash separator.""" mock_arn = fake_arn(resource_separator="/") arn_object = utils.AwsArn(mock_arn) resource_type = arn_object.resource_type self.assertIsNotNone(resource_type) resource_separator = arn_object.resource_separator self.assertEqual(resource_separator, "/") resource = arn_object.resource self.assertIsNotNone(resource)
def test_download_file_raise_nofile_err(self, fake_session): """Test that downloading a nonexistent file fails with AWSReportDownloaderNoFileError.""" fake_response = {"Error": {"Code": "NoSuchKey"}} fake_client = Mock() fake_client.get_object.side_effect = ClientError(fake_response, "masu-test") auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) downloader.s3_client = fake_client with self.assertRaises(AWSReportDownloaderNoFileError): downloader.download_file(self.fake.file_path())
def test_download_file_raise_downloader_err(self, fake_session): """Test _check_size fails when there is a downloader error.""" fake_response = {"Error": {"Code": self.fake.word()}} fake_client = Mock() fake_client.get_object.side_effect = ClientError(fake_response, "masu-test") auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) downloader.s3_client = fake_client with self.assertRaises(AWSReportDownloaderError): downloader.download_file(self.fake.file_path())
def test_check_size_fail_nosize(self, fake_session): """Test _check_size fails if there report has no size.""" fake_client = Mock() fake_client.get_object.return_value = {} auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) downloader.s3_client = fake_client fakekey = self.fake.file_path(depth=random.randint(1, 5), extension=random.choice(["json", "csv.gz"])) with self.assertRaises(AWSReportDownloaderError): downloader._check_size(fakekey, check_inflate=False)
def test_get_report_update_status(self, fake_downloader, fake_status): """Test that status is updated when downloading is complete.""" account = fake_arn(service="iam", generate_account_id=True) _get_report_files( Mock(), customer_name=self.fake.word(), authentication=account, provider_type=Provider.PROVIDER_AWS, report_month=DateAccessor().today(), provider_uuid=self.aws_provider_uuid, billing_source=self.fake.word(), ) fake_status.assert_called_with(ProviderStatusCode.READY)