def test_update_summary_tables_aws_end_date(self, mock_charge_info):
"""Test that the summary table task respects a date range."""
provider = Provider.PROVIDER_AWS_LOCAL
provider_aws_uuid = self.aws_provider_uuid
ce_table_name = AWS_CUR_TABLE_MAP["cost_entry"]
daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"]
summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"]
start_date = DateHelper().last_month_start
end_date = DateHelper().last_month_end
daily_table = getattr(self.aws_accessor.report_schema,
daily_table_name)
summary_table = getattr(self.aws_accessor.report_schema,
summary_table_name)
ce_table = getattr(self.aws_accessor.report_schema, ce_table_name)
with schema_context(self.schema):
daily_table.objects.all().delete()
summary_table.objects.all().delete()
ce_start_date = ce_table.objects.filter(
interval_start__gte=start_date.date()).aggregate(
Min("interval_start"))["interval_start__min"]
ce_end_date = ce_table.objects.filter(
interval_start__lte=end_date.date()).aggregate(
Max("interval_start"))["interval_start__max"]
# The summary tables will only include dates where there is data
expected_start_date = max(start_date, ce_start_date)
expected_start_date = expected_start_date.replace(hour=0,
minute=0,
second=0,
microsecond=0)
expected_end_date = min(end_date, ce_end_date)
expected_end_date = expected_end_date.replace(hour=0,
minute=0,
second=0,
microsecond=0)
update_summary_tables(self.schema, provider, provider_aws_uuid,
start_date, end_date)
with schema_context(self.schema):
daily_entry = daily_table.objects.all().aggregate(
Min("usage_start"), Max("usage_end"))
result_start_date = daily_entry["usage_start__min"]
result_end_date = daily_entry["usage_end__max"]
self.assertEqual(result_start_date, expected_start_date.date())
self.assertEqual(result_end_date, expected_end_date.date())
with schema_context(self.schema):
summary_entry = summary_table.objects.all().aggregate(
Min("usage_start"), Max("usage_end"))
result_start_date = summary_entry["usage_start__min"]
result_end_date = summary_entry["usage_end__max"]
self.assertEqual(result_start_date, expected_start_date.date())
self.assertEqual(result_end_date, expected_end_date.date())
def test_azure_date_converter(self):
"""Test that we convert the new Azure date format."""
today = DateHelper().today
old_azure_format = today.strftime("%Y-%m-%d")
new_azure_format = today.strftime("%m/%d/%Y")
self.assertEqual(
azure_date_converter(old_azure_format).date(), today.date())
self.assertEqual(
azure_date_converter(new_azure_format).date(), today.date())
def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate,
mock_mem_rate,
mock_charge_info):
"""Test that the summary table task respects a date range."""
mock_cpu_rate.return_value = 1.5
mock_mem_rate.return_value = 2.5
provider = Provider.PROVIDER_OCP
provider_ocp_uuid = self.ocp_test_provider_uuid
ce_table_name = OCP_REPORT_TABLE_MAP["report"]
daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"]
start_date = DateHelper().last_month_start
end_date = DateHelper().last_month_end
daily_table = getattr(self.ocp_accessor.report_schema,
daily_table_name)
ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name)
with schema_context(self.schema):
daily_table.objects.all().delete()
ce_start_date = ce_table.objects.filter(
interval_start__gte=start_date.date()).aggregate(
Min("interval_start"))["interval_start__min"]
ce_end_date = ce_table.objects.filter(
interval_start__lte=end_date.date()).aggregate(
Max("interval_start"))["interval_start__max"]
# The summary tables will only include dates where there is data
expected_start_date = max(start_date, ce_start_date)
expected_end_date = min(end_date, ce_end_date)
update_summary_tables(self.schema, provider, provider_ocp_uuid,
start_date, end_date)
with schema_context(self.schema):
daily_entry = daily_table.objects.all().aggregate(
Min("usage_start"), Max("usage_end"))
result_start_date = daily_entry["usage_start__min"]
result_end_date = daily_entry["usage_end__max"]
self.assertEqual(result_start_date, expected_start_date.date())
self.assertEqual(result_end_date, expected_end_date.date())
def test_get_report_data_with_end_date(self, mock_update, mock_accessor,
_):
"""Test GET report_data endpoint with end date."""
start_date = DateHelper().today
end_date = start_date + datetime.timedelta(days=1)
multiple_calls = start_date.month != end_date.month
provider_type = Provider.PROVIDER_AWS
mock_accessor.return_value.__enter__.return_value.get_type.return_value = provider_type
params = {
"schema": "acct10001",
"provider_uuid": "6e212746-484a-40cd-bba0-09a19d132d64",
"start_date": start_date.date().strftime("%Y-%m-%d"),
"end_date": end_date.date().strftime("%Y-%m-%d"),
}
expected_key = "Report Data Task IDs"
expected_calls = [
call(
params["schema"],
provider_type,
params["provider_uuid"],
params["start_date"],
params["end_date"],
queue_name=PRIORITY_QUEUE,
)
]
if multiple_calls:
expected_calls = [
call(
params["schema"],
provider_type,
params["provider_uuid"],
params["start_date"],
params["start_date"],
queue_name=PRIORITY_QUEUE,
),
call(
params["schema"],
provider_type,
params["provider_uuid"],
params["end_date"],
params["end_date"],
queue_name=PRIORITY_QUEUE,
),
]
response = self.client.get(reverse("report_data"), params)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIn(expected_key, body)
mock_update.s.assert_has_calls(expected_calls, any_order=True)
def test_create_bill_with_string_arg(self):
"""Test that a bill is created in the Postgres database."""
bill_date = DateHelper().this_month_start
start_date = bill_date
end_date = DateHelper().this_month_end
self.processor.create_bill(str(bill_date.date()))
with schema_context(self.schema):
bill = AzureCostEntryBill.objects.filter(
billing_period_start=start_date, billing_period_end=end_date, provider=self.azure_provider_uuid
)
self.assertIsNotNone(bill.first())
def test_create_bill_with_string_arg(self, mock_execute_sql):
"""Test that a bill is created in the Postgres database."""
bill_date = DateHelper().this_month_start
start_date = bill_date
end_date = DateHelper().this_month_end
account_id = "9999999999"
mock_execute_sql.return_value = [[account_id]]
self.processor.create_bill(str(bill_date.date()))
with schema_context(self.schema):
bill = GCPCostEntryBill.objects.filter(
billing_period_start=start_date, billing_period_end=end_date, provider=self.gcp_provider
)
self.assertIsNotNone(bill.first())
def test_get_report_data_with_only_provider_type(self, mock_update, _):
"""Test GET report_data endpoint with only provider_type."""
start_date = DateHelper().today
end_date = start_date + datetime.timedelta(days=1)
multiple_calls = start_date.month != end_date.month
params = {
"schema": "acct10001",
"provider_type": Provider.PROVIDER_AWS,
"start_date": start_date.date().strftime("%Y-%m-%d"),
"end_date": end_date.date().strftime("%Y-%m-%d"),
}
expected_key = "Report Data Task IDs"
expected_calls = [
call(
params["schema"],
params["provider_type"],
None,
params["start_date"],
params["end_date"],
queue_name=PRIORITY_QUEUE,
)
]
if multiple_calls:
expected_calls = [
call(
params["schema"],
params["provider_type"],
None,
params["start_date"],
params["start_date"],
queue_name=PRIORITY_QUEUE,
),
call(
params["schema"],
params["provider_type"],
None,
params["end_date"],
params["end_date"],
queue_name=PRIORITY_QUEUE,
),
]
response = self.client.get(reverse("report_data"), params)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIn(expected_key, body)
mock_update.s.assert_has_calls(expected_calls, any_order=True)
def test_create_bill(self):
"""Test that a bill is created in the Postgres database."""
bill_date = DateHelper().next_month_start
start_date = bill_date
end_date = DateHelper().next_month_end
self.processor.create_bill(bill_date.date())
with schema_context(self.schema):
report_period = OCPUsageReportPeriod.objects.filter(
cluster_id=self.ocp_cluster_id,
report_period_start=start_date,
report_period_end=end_date,
provider=self.ocp_provider_uuid,
)
self.assertIsNotNone(report_period.first())
def test_get_report_data(self, mock_celery, mock_accessor, _):
"""Test the GET report_data endpoint."""
provider_type = Provider.PROVIDER_AWS
mock_accessor.return_value.__enter__.return_value.get_type.return_value = provider_type
end_date = DateHelper().today
start_date = end_date - timedelta(days=1)
params = {
"schema": "acct10001",
"start_date": start_date.date().strftime("%Y-%m-%d"),
"end_date": end_date.date().strftime("%Y-%m-%d"),
"provider_uuid": "6e212746-484a-40cd-bba0-09a19d132d64",
"provider": "AWS",
"tracing_id": str(uuid.uuid4()),
}
expected_key = "HCS Report Data Task ID"
response = self.client.get(reverse(self.ENDPOINT), params)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIn(expected_key, body)
mock_celery.s.assert_called()
class GCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the GCPReportDownloader object."""
def setUp(self):
"""Setup vars for test."""
super().setUp()
self.etag = "1234"
self.today = DateHelper().today
def tearDown(self):
"""Remove files and directories created during the test run."""
super().tearDown()
shutil.rmtree(DATA_DIR, ignore_errors=True)
def create_gcp_downloader_with_mocked_values(
self,
customer_name=FAKE.name(),
dataset=FAKE.slug(),
provider_uuid=uuid4(),
project_id=FAKE.slug(),
table_id=FAKE.slug(),
):
"""
Create a GCPReportDownloader that skips the initial GCP bigquery check creates etag.
This also results in Mock objects being set to instance variables that can be patched
inside other test functions.
Args:
customer_name (str): optional customer name; will be randomly generated if None
bucket_name (str): optional bucket name; will be randomly generated if None
provider_uuid (uuid): optional provider UUID; will be randomly generated if None
Returns:
GCPReportDownloader instance with faked argument data and Mocks in
self.etag.
"""
billing_source = {"table_id": table_id, "dataset": dataset}
credentials = {"project_id": project_id}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag",
return_value=self.etag,
):
downloader = GCPReportDownloader(
customer_name=customer_name,
data_source=billing_source,
provider_uuid=provider_uuid,
credentials=credentials,
)
return downloader
@patch("masu.external.downloader.gcp.gcp_report_downloader.GCPProvider")
def test_generate_etag_big_query_client_error(self, gcp_provider):
"""Test BigQuery client is handled correctly in generate etag method."""
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
err_msg = "GCP Error"
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.bigquery"
) as bigquery:
bigquery.Client.side_effect = GoogleCloudError(err_msg)
with self.assertRaisesRegex(ReportDownloaderWarning, err_msg):
GCPReportDownloader(
customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials,
)
@patch("masu.external.downloader.gcp.gcp_report_downloader.GCPProvider")
def test_generate_etag(self, gcp_provider):
"""Test BigQuery client is handled correctly in generate etag method."""
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.bigquery"
) as bigquery:
bigquery.Client.return_value.get_table.return_value.modified.return_value = self.today
downloader = GCPReportDownloader(customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials)
self.assertIsNotNone(downloader.etag)
@patch("masu.external.downloader.gcp.gcp_report_downloader.os.makedirs")
@patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery")
def test_download_file_failure_on_file_open(self, mock_bigquery,
mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
key = "202011_1234_2020-12-05:2020-12-08.csv"
downloader = self.create_gcp_downloader_with_mocked_values()
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"
) as mock_open:
err_msg = "bad open"
mock_open.side_effect = IOError(err_msg)
with self.assertRaisesRegex(GCPReportDownloaderError, err_msg):
downloader.download_file(key)
def test_generate_monthly_pseudo_manifest(self):
"""Assert _generate_monthly_pseudo_manifest returns a manifest-like dict."""
provider_uuid = uuid4()
dh = DateHelper()
start_date = dh.this_month_start
invoice_month = start_date.strftime("%Y%m")
expected_assembly_id = ":".join(
[str(provider_uuid), self.etag, invoice_month])
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=provider_uuid)
downloader.scan_end = dh.this_month_end.date()
if self.today.date() < downloader.scan_end:
expected_end_date = self.today.date()
else:
expected_end_date = downloader.scan_end
result_manifest = downloader._generate_monthly_pseudo_manifest(
start_date.date())
expected_files = create_expected_csv_files(dh.this_month_start.date(),
downloader.scan_end,
invoice_month, self.etag)
expected_manifest_data = {
"assembly_id": expected_assembly_id,
"compression": UNCOMPRESSED,
"start_date": start_date.date(),
"end_date": expected_end_date, # inclusive end date
"file_names": expected_files,
}
self.assertEqual(result_manifest, expected_manifest_data)
def test_generate_assembly_id(self):
"""Assert appropriate generation of assembly ID."""
provider_uuid = uuid4()
expected_assembly_id = ":".join([str(provider_uuid), self.etag, "1"])
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=provider_uuid)
assembly_id = downloader._generate_assembly_id("1")
self.assertEqual(assembly_id, expected_assembly_id)
def test_relevant_file_names(self):
"""Assert relevant file name is generated correctly."""
downloader = self.create_gcp_downloader_with_mocked_values()
mock_invoice_month = self.today.strftime("%Y%m")
end_date = downloader.scan_end + relativedelta(days=1)
expected_file_name = [
f"{mock_invoice_month}_{self.etag}_{downloader.scan_start}:{end_date}.csv"
]
result_file_names = downloader._get_relevant_file_names(
mock_invoice_month)
self.assertEqual(expected_file_name, result_file_names)
def test_get_local_file_for_report(self):
"""Assert that get_local_file_for_report is a simple pass-through."""
downloader = self.create_gcp_downloader_with_mocked_values()
report_name = FAKE.file_path()
local_name = downloader.get_local_file_for_report(report_name)
self.assertEqual(local_name, report_name)
@patch("masu.external.downloader.gcp.gcp_report_downloader.os.makedirs")
@patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery")
def test_download_file_success(self, mock_bigquery, mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
key = "202011_1234_2020-12-05:2020-12-08.csv"
mock_name = "mock-test-customer-success"
expected_full_path = f"{DATA_DIR}/{mock_name}/gcp/{key}"
downloader = self.create_gcp_downloader_with_mocked_values(
customer_name=mock_name)
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.create_daily_archives"
):
full_path, etag, date, _ = downloader.download_file(key)
mock_makedirs.assert_called()
self.assertEqual(etag, self.etag)
self.assertEqual(date, self.today)
self.assertEqual(full_path, expected_full_path)
@patch("masu.external.downloader.gcp.gcp_report_downloader.os.makedirs")
@patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery")
def test_download_file_success_end_date_today(self, mock_bigquery,
mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
end_date = DateAccessor().today().date()
key = f"202011_1234_2020-12-05:{end_date}.csv"
mock_name = "mock-test-customer-end-date"
expected_full_path = f"{DATA_DIR}/{mock_name}/gcp/{key}"
downloader = self.create_gcp_downloader_with_mocked_values(
customer_name=mock_name)
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.create_daily_archives"
):
full_path, etag, date, _ = downloader.download_file(key)
mock_makedirs.assert_called()
self.assertEqual(etag, self.etag)
self.assertEqual(date, self.today)
self.assertEqual(full_path, expected_full_path)
@patch("masu.external.downloader.gcp.gcp_report_downloader.open")
def test_download_file_query_client_error(self, mock_open):
"""Test BigQuery client is handled correctly in download file method."""
key = "202011_1234_2020-12-05:2020-12-08.csv"
downloader = self.create_gcp_downloader_with_mocked_values()
err_msg = "GCP Error"
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.bigquery"
) as bigquery:
bigquery.Client.side_effect = GoogleCloudError(err_msg)
with self.assertRaisesRegex(GCPReportDownloaderError, err_msg):
downloader.download_file(key)
@patch("masu.external.downloader.gcp.gcp_report_downloader.GCPProvider")
def test_download_with_unreachable_source(self, gcp_provider):
"""Assert errors correctly when source is unreachable."""
gcp_provider.return_value.cost_usage_source_is_reachable.side_effect = ValidationError
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
with self.assertRaises(GCPReportDownloaderError):
GCPReportDownloader(FAKE.name(),
billing_source,
credentials=credentials)
def test_get_manifest_context_for_date(self):
"""Test successful return of get manifest context for date."""
self.maxDiff = None
dh = DateHelper()
start_date = dh.this_month_start
invoice_month = start_date.strftime("%Y%m")
p_uuid = uuid4()
expected_assembly_id = f"{p_uuid}:{self.etag}:{invoice_month}"
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=p_uuid)
if self.today.date() < downloader.scan_end:
expected_end = self.today.date()
else:
expected_end = downloader.scan_end
expected_files = create_expected_csv_files(dh.this_month_start.date(),
expected_end, invoice_month,
self.etag, True)
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._process_manifest_db_record",
return_value=2,
):
report_dict = downloader.get_manifest_context_for_date(
start_date.date())
self.assertEqual(report_dict.get("manifest_id"), 2)
self.assertEqual(report_dict.get("files"), expected_files)
self.assertEqual(report_dict.get("compression"), UNCOMPRESSED)
self.assertEqual(report_dict.get("assembly_id"), expected_assembly_id)
def test_generate_monthly_pseudo_no_manifest(self):
"""Test get monly psuedo manifest with no manifest."""
dh = DateHelper()
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=uuid4())
start_date = dh.last_month_start
manifest_dict = downloader._generate_monthly_pseudo_manifest(
start_date)
self.assertIsNotNone(manifest_dict)
@override_settings(ENABLE_PARQUET_PROCESSING=True)
@patch(
"masu.external.downloader.gcp.gcp_report_downloader.copy_local_report_file_to_s3_bucket"
)
def test_create_daily_archives(self, mock_s3):
"""Test that we load daily files to S3."""
# Use the processor example for data:
file_path = "./koku/masu/test/data/gcp/202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
file_name = "202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
temp_dir = tempfile.gettempdir()
temp_path = os.path.join(temp_dir, file_name)
shutil.copy2(file_path, temp_path)
expected_daily_files = [
f"{temp_dir}/202011_2020-11-08.csv",
f"{temp_dir}/202011_2020-11-09.csv",
f"{temp_dir}/202011_2020-11-10.csv",
f"{temp_dir}/202011_2020-11-11.csv",
]
start_date = DateHelper().this_month_start
daily_file_names = create_daily_archives("request_id", "account",
self.gcp_provider_uuid,
file_name, temp_path, None,
start_date, None)
mock_s3.assert_called()
self.assertEqual(sorted(daily_file_names),
sorted(expected_daily_files))
for daily_file in expected_daily_files:
self.assertTrue(os.path.exists(daily_file))
os.remove(daily_file)
os.remove(temp_path)
def test_get_dataset_name(self):
"""Test _get_dataset_name helper."""
project_id = FAKE.slug()
dataset_name = FAKE.slug()
datasets = [f"{project_id}:{dataset_name}", dataset_name]
for dataset in datasets:
billing_source = {"table_id": FAKE.slug(), "dataset": dataset}
credentials = {"project_id": project_id}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag",
return_value=self.etag,
):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
):
downloader = GCPReportDownloader(
customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials,
)
self.assertEqual(downloader._get_dataset_name(), dataset_name)
