def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSLocalReportDownloader(
self.fake_customer_name,
auth_credential,
self.fake_bucket_name,
provider_uuid=self.aws_provider_uuid)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = "GZIP"
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {
"start": start_str
},
},
)
result = downloader.get_manifest_context_for_date(current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
def test_get_report_context_for_date_should_not_download(
self, mock_session, mock_manifest, mock_delete, mock_check
):
"""Test that no data is returned when we don't want to process."""
current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = downloader.report.get("Compression")
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {"start": start_str},
},
)
mock_check.return_value = False
expected = {}
result = downloader.get_report_context_for_date(current_month)
self.assertEqual(result, expected)
def setUp(self):
"""Set up shared variables."""
super().setUp()
self.aws_credentials = self.aws_provider.authentication.credentials
self.aws_data_source = self.aws_provider.billing_source.data_source
self.azure_credentials = self.azure_provider.authentication.credentials
self.azure_data_source = self.azure_provider.billing_source.data_source
self.gcp_credentials = self.gcp_provider.authentication.credentials
self.gcp_data_source = self.gcp_provider.billing_source.data_source
self.ocp_credentials = [
name[0] for name in Provider.objects.values_list(
"authentication__credentials")
]
self.ocp_data_source = {}
self.mock_accounts = [{
"credentials": {
"role_arn": fake_arn(service="iam", generate_account_id=True)
},
"data_source": {
"bucket": self.fake.word()
},
"customer_name": self.fake.word(),
"provider_type": Provider.PROVIDER_AWS,
"schema_name": self.fake.word(),
}]
def setUp(self):
"""Set up the test."""
super().setUp()
self.account_id = fake_aws_account_id()
self.arn = fake_arn(account_id=self.account_id,
region=REGION,
service="iam")
def test_download_default_report_no_report_found(self, fake_session,
fake_report_list):
auth_credential = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.mock_task, self.fake_customer_name,
auth_credential, self.fake_bucket_name)
def test_missing_report_name(self, fake_session):
"""Test downloading a report with an invalid report name."""
auth_credential = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.mock_task, self.fake_customer_name,
auth_credential, 's3_bucket', 'wrongreport')
def setUp(self):
"""Set up test case."""
super().setUp()
self.schema = "acct12345"
self.paginator_dict = {
"r-0": {
"OrganizationalUnits": [
{"Id": "ou-0", "Arn": "arn-0", "Name": "Big_Org_0"},
{"Id": "ou-1", "Arn": "arn-1", "Name": "Big_Org_1"},
{"Id": "ou-2", "Arn": "arn-2", "Name": "Big_Org_2"},
]
},
"ou-0": {"OrganizationalUnits": [{"Id": "sou-0", "Arn": "arn-0", "Name": "Sub_Org_0"}]},
"ou-1": {"OrganizationalUnits": []},
"ou-2": {"OrganizationalUnits": []},
"sou-0": {"OrganizationalUnits": []},
}
self.account = {
"credentials": {"role_arn": fake_arn(service="iam", generate_account_id=True)},
"customer_name": CUSTOMER_NAME,
"billing_source": BUCKET,
"provider_type": P_TYPE,
"schema_name": self.schema,
"provider_uuid": P_UUID,
}
def setUp(self):
"""Set up shared variables."""
super().setUp()
self.aws_provider_resource_name = self.aws_provider.authentication.provider_resource_name
self.aws_billing_source = self.aws_provider.billing_source.bucket
self.azure_credentials = self.azure_provider.authentication.credentials
self.azure_data_source = self.azure_provider.billing_source.data_source
self.ocp_provider_resource_names = [
name[0] for name in Provider.objects.values_list(
"authentication__provider_resource_name")
]
self.ocp_billing_source = None
self.mock_accounts = []
self.mock_accounts.append({
"authentication":
fake_arn(service="iam", generate_account_id=True),
"billing_source":
self.fake.word(),
"customer_name":
self.fake.word(),
"provider_type":
Provider.PROVIDER_AWS,
"schema_name":
self.fake.word(),
})
def test_download_default_report(self, fake_session):
# actual test
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
self.assertEqual(downloader.report_name, self.fake_report_name)
def setUp(self):
"""Set up test case."""
super().setUp()
self.paginator_dict = {
"r-0": {
"OrganizationalUnits": [
{"Id": "ou-0", "Arn": "arn-0", "Name": "Big_Org_0"},
{"Id": "ou-1", "Arn": "arn-1", "Name": "Big_Org_1"},
{"Id": "ou-2", "Arn": "arn-2", "Name": "Big_Org_2"},
]
},
"ou-0": {"OrganizationalUnits": [{"Id": "sou-0", "Arn": "arn-0", "Name": "Sub_Org_0"}]},
"ou-1": {"OrganizationalUnits": []},
"ou-2": {"OrganizationalUnits": []},
"sou-0": {"OrganizationalUnits": []},
}
self.account = {
"credentials": {"role_arn": fake_arn(service="iam", generate_account_id=True)},
"customer_name": CUSTOMER_NAME,
"billing_source": BUCKET,
"provider_type": P_TYPE,
"schema_name": self.schema,
"provider_uuid": P_UUID,
}
with schema_context(self.schema):
# Delete the rows created by the koku_test_runner. This next test suite
# is intended to test how the crawler inserts the data into the database
# which is easier to do without preloaded data.
AWSOrganizationalUnit.objects.all().delete()
def test_parse_arn_with_region_and_account(self):
"""Assert successful account ID parsing from a well-formed ARN."""
mock_account_id = fake_aws_account_id()
mock_arn = fake_arn(account_id=mock_account_id, region="test-region-1")
arn_object = utils.AwsArn(mock_arn)
partition = arn_object.partition
self.assertIsNotNone(partition)
service = arn_object.service
self.assertIsNotNone(service)
region = arn_object.region
self.assertIsNotNone(region)
account_id = arn_object.account_id
self.assertIsNotNone(account_id)
resource_type = arn_object.resource_type
self.assertIsNotNone(resource_type)
resource_separator = arn_object.resource_separator
self.assertIsNotNone(resource_separator)
resource = arn_object.resource
self.assertIsNotNone(resource)
reconstructed_arn = ("arn:" + partition + ":" + service + ":" +
region + ":" + account_id + ":" + resource_type +
resource_separator + resource)
self.assertEqual(mock_account_id, account_id)
self.assertEqual(mock_arn, reconstructed_arn)
def test_init_with_demo_account(self):
"""Test init with the demo account."""
mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
account_id = "123456"
arn = fake_arn(service="iam", generate_account_id=True)
credentials = {"role_arn": arn}
report_name = FAKE.word()
demo_accounts = {
account_id: {
arn: {
"report_name": report_name,
"report_prefix": FAKE.word()
}
}
}
with self.settings(DEMO_ACCOUNTS=demo_accounts):
with patch("masu.util.aws.common.get_assume_role_session"
) as mock_session:
AWSReportDownloader(
**{
"task": mock_task,
"customer_name": f"acct{account_id}",
"credentials": credentials,
"data_source": {
"bucket": FAKE.word()
},
"report_name": report_name,
"provider_uuid": self.aws_provider_uuid,
})
mock_session.assert_called_once()
def test_get_report_context_for_date_should_not_download(
self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that no data is returned when we don't want to process."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = '1234'
compression = downloader.report.get('Compression')
report_keys = ['file1', 'file2']
mock_manifest.return_value = ('', {
'assemblyId': assembly_id,
'Compression': compression,
'reportKeys': report_keys,
'billingPeriod': {
'start': start_str
}
})
mock_check.return_value = False
expected = {}
result = downloader.get_report_context_for_date(current_month)
self.assertEqual(result, expected)
def test_disk_status_logging(self, fake_downloader):
"""Test task for logging when temp directory exists."""
logging.disable(logging.NOTSET)
os.makedirs(Config.TMP_DIR, exist_ok=True)
account = fake_arn(service="iam", generate_account_id=True)
expected = "INFO:masu.processor._tasks.download:Available disk space"
with self.assertLogs("masu.processor._tasks.download",
level="INFO") as logger:
_get_report_files(
Mock(),
customer_name=self.fake.word(),
authentication=account,
provider_type=Provider.PROVIDER_AWS,
report_month=DateHelper().today,
provider_uuid=self.aws_provider_uuid,
billing_source=self.fake.word(),
cache_key=self.fake.word(),
)
statement_found = False
for log in logger.output:
if expected in log:
statement_found = True
self.assertTrue(statement_found)
shutil.rmtree(Config.TMP_DIR, ignore_errors=True)
def test_download_default_report_no_report_found(self, fake_session,
fake_report_list):
"""Test download fails when no reports are found."""
auth_credential = fake_arn(service="iam", generate_account_id=True)
with self.assertRaises(MasuProviderError):
AWSReportDownloader(self.fake_customer_name, auth_credential,
self.fake_bucket_name)
def test_download_default_report(self, fake_session):
"""Test assume aws role works."""
# actual test
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
self.assertEqual(downloader.report_name, self.fake_report_name)
def setUp(self):
"""Set up the test."""
super().setUp()
self.account_id = fake_aws_account_id()
self.arn = fake_arn(account_id=self.account_id,
region=REGION,
service='iam')
with ReportingCommonDBAccessor() as common_accessor:
self.column_map = common_accessor.column_map
def setUpClass(cls):
super().setUpClass()
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = REPORT
cls.fake_bucket_prefix = PREFIX
cls.fake_bucket_name = BUCKET
cls.selected_region = REGION
cls.auth_credential = fake_arn(service='iam', generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
def test_parse_arn_without_region_or_account(self):
"""Assert successful ARN parsing without a region or an account id."""
mock_arn = fake_arn()
arn_object = utils.AwsArn(mock_arn)
region = arn_object.region
self.assertEqual(region, None)
account_id = arn_object.account_id
self.assertEqual(account_id, None)
def setUpClass(cls):
"""Set up class variables."""
super().setUpClass()
cls.fake_customer_name = CUSTOMER_NAME
cls.fake_report_name = "koku-local"
cls.fake_bucket_prefix = PREFIX
cls.selected_region = REGION
cls.fake_auth_credential = fake_arn(service="iam", generate_account_id=True)
cls.manifest_accessor = ReportManifestDBAccessor()
def setUpClass(cls):
"""Set up the class."""
super().setUpClass()
cls.fake = faker.Faker()
cls.fake_reports = [
{'file': cls.fake.word(), 'compression': 'GZIP'},
{'file': cls.fake.word(), 'compression': 'PLAIN'},
]
cls.fake_account = fake_arn(service='iam', generate_account_id=True)
cls.today = DateAccessor().today_with_timezone('UTC')
cls.yesterday = cls.today - timedelta(days=1)
def setUpClass(cls):
"""Set up the class."""
super().setUpClass()
cls.fake = faker.Faker()
cls.fake_reports = [
{"file": cls.fake.word(), "compression": "GZIP"},
{"file": cls.fake.word(), "compression": "PLAIN"},
]
cls.fake_account = fake_arn(service="iam", generate_account_id=True)
cls.today = DateAccessor().today_with_timezone("UTC")
cls.yesterday = cls.today - timedelta(days=1)
def test_download_file_raise_nofile_err(self, fake_session):
fake_response = {'Error': {'Code': 'NoSuchKey'}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(fake_response, 'masu-test')
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderNoFileError):
downloader.download_file(self.fake.file_path())
def test_get_report_exception(self, fake_downloader):
"""Test task"""
account = fake_arn(service='iam', generate_account_id=True)
with self.assertRaises(Exception):
_get_report_files(
customer_name=self.fake.word(),
authentication=account,
provider_type='AWS',
report_name=self.fake.word(),
provider_uuid=self.aws_test_provider_uuid,
billing_source=self.fake.word(),
)
def test_get_report_update_status(self, fake_downloader, fake_status):
"""Test that status is updated when downloading is complete."""
account = fake_arn(service='iam', generate_account_id=True)
_get_report_files(
customer_name=self.fake.word(),
authentication=account,
provider_type='AWS',
report_name=self.fake.word(),
provider_uuid=self.aws_test_provider_uuid,
billing_source=self.fake.word(),
)
fake_status.assert_called_with(ProviderStatusCode.READY)
def test_parse_arn_with_slash_separator(self):
"""Assert successful ARN parsing with a slash separator."""
mock_arn = fake_arn(resource_separator="/")
arn_object = utils.AwsArn(mock_arn)
resource_type = arn_object.resource_type
self.assertIsNotNone(resource_type)
resource_separator = arn_object.resource_separator
self.assertEqual(resource_separator, "/")
resource = arn_object.resource
self.assertIsNotNone(resource)
def test_download_file_raise_nofile_err(self, fake_session):
"""Test that downloading a nonexistent file fails with AWSReportDownloaderNoFileError."""
fake_response = {"Error": {"Code": "NoSuchKey"}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(fake_response, "masu-test")
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderNoFileError):
downloader.download_file(self.fake.file_path())
def test_download_file_raise_downloader_err(self, fake_session):
"""Test _check_size fails when there is a downloader error."""
fake_response = {"Error": {"Code": self.fake.word()}}
fake_client = Mock()
fake_client.get_object.side_effect = ClientError(fake_response, "masu-test")
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
with self.assertRaises(AWSReportDownloaderError):
downloader.download_file(self.fake.file_path())
def test_check_size_fail_nosize(self, fake_session):
"""Test _check_size fails if there report has no size."""
fake_client = Mock()
fake_client.get_object.return_value = {}
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
downloader.s3_client = fake_client
fakekey = self.fake.file_path(depth=random.randint(1, 5), extension=random.choice(["json", "csv.gz"]))
with self.assertRaises(AWSReportDownloaderError):
downloader._check_size(fakekey, check_inflate=False)
def test_get_report_update_status(self, fake_downloader, fake_status):
"""Test that status is updated when downloading is complete."""
account = fake_arn(service="iam", generate_account_id=True)
_get_report_files(
Mock(),
customer_name=self.fake.word(),
authentication=account,
provider_type=Provider.PROVIDER_AWS,
report_month=DateAccessor().today(),
provider_uuid=self.aws_provider_uuid,
billing_source=self.fake.word(),
)
fake_status.assert_called_with(ProviderStatusCode.READY)
