def test_update_summary_tables_without_manifest(self, mock_daily,
mock_summary):
"""Test that summary tables are properly run without a manifest."""
self.updater = AWSReportSummaryUpdater('acct10001', self.provider,
None)
start_date = DateAccessor().today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.summary_data_updated_datetime = start_date
self.accessor.commit()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
with AWSReportDBAccessor('acct10001', self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertGreater(bill.summary_data_updated_datetime, start_date)
def summarize_reports(reports_to_summarize):
"""
Summarize reports returned from line summary task.
Args:
reports_to_summarize (list) list of reports to process
Returns:
None
"""
for report in reports_to_summarize:
# For day-to-day summarization we choose a small window to
# cover new data from a window of days.
# This saves us from re-summarizing unchanged data and cuts down
# on processing time. There are override mechanisms in the
# Updater classes for when full-month summarization is
# required.
start_date = DateAccessor().today() - datetime.timedelta(days=2)
start_date = start_date.strftime("%Y-%m-%d")
end_date = DateAccessor().today().strftime("%Y-%m-%d")
LOG.info("report to summarize: %s", str(report))
update_summary_tables.delay(
report.get("schema_name"),
report.get("provider_type"),
report.get("provider_uuid"),
start_date=start_date,
end_date=end_date,
manifest_id=report.get("manifest_id"),
)
def test_get_manifest_context_for_date(self, mock_manifest):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
start_str = current_month.strftime(
self.downloader.manifest_date_format)
assembly_id = "1234"
compression = UNCOMPRESSED
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {
"start": start_str
},
},
DateAccessor().today(),
)
result = self.downloader.get_manifest_context_for_date(current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
def setUpClass(cls):
"""Set up the test class."""
super().setUpClass()
today = DateAccessor().today_with_timezone("UTC")
cls.today = today.strftime("%Y-%m-%d")
cls.tomorrow = (today +
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
def test_get_report_context_for_date_should_not_download(
self, mock_session, mock_manifest, mock_delete, mock_check
):
"""Test that no data is returned when we don't want to process."""
current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name
)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = downloader.report.get("Compression")
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {"start": start_str},
},
)
mock_check.return_value = False
expected = {}
result = downloader.get_report_context_for_date(current_month)
self.assertEqual(result, expected)
def test_get_manifest_context_for_date(self, mock_session, mock_manifest,
mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
downloader = AWSReportDownloader(self.fake_customer_name,
self.credentials,
self.data_source,
provider_uuid=self.aws_provider_uuid)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = downloader.report.get("Compression")
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {
"start": start_str
},
},
DateAccessor().today(),
)
result = downloader.get_manifest_context_for_date(current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
def test_get_report_context_for_date_should_not_download(
self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that no data is returned when we don't want to process."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = '1234'
compression = downloader.report.get('Compression')
report_keys = ['file1', 'file2']
mock_manifest.return_value = ('', {
'assemblyId': assembly_id,
'Compression': compression,
'reportKeys': report_keys,
'billingPeriod': {
'start': start_str
}
})
mock_check.return_value = False
expected = {}
result = downloader.get_report_context_for_date(current_month)
self.assertEqual(result, expected)
def setUpClass(cls):
"""Set up the test class."""
super().setUpClass()
cls.schema = 'acct10001'
today = DateAccessor().today_with_timezone('UTC')
cls.today = today.strftime('%Y-%m-%d')
cls.tomorrow = (today +
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
def test_update_summary_tables_without_manifest(
self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary
):
"""Test that summary tables are properly run without a manifest."""
# Create an updater that doesn't have a manifest
updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
start_date = DateAccessor().today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_updated_datetime = start_date
period.save()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
mock_storage_daily.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
mock_storage_summary.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertGreater(period.summary_data_updated_datetime, start_date)
def summarize_reports(reports_to_summarize, queue_name=None):
"""
Summarize reports returned from line summary task.
Args:
reports_to_summarize (list) list of reports to process
Returns:
None
"""
reports_to_summarize = [
report for report in reports_to_summarize if report
]
reports_deduplicated = [
dict(t) for t in {tuple(d.items())
for d in reports_to_summarize}
]
for report in reports_deduplicated:
# For day-to-day summarization we choose a small window to
# cover new data from a window of days.
# This saves us from re-summarizing unchanged data and cuts down
# on processing time. There are override mechanisms in the
# Updater classes for when full-month summarization is
# required.
with ReportManifestDBAccessor() as manifest_accesor:
if manifest_accesor.manifest_ready_for_summary(
report.get("manifest_id")):
if report.get("start") and report.get("end"):
LOG.info("using start and end dates from the manifest")
start_date = parser.parse(
report.get("start")).strftime("%Y-%m-%d")
end_date = parser.parse(
report.get("end")).strftime("%Y-%m-%d")
else:
LOG.info("generating start and end dates for manifest")
start_date = DateAccessor().today() - datetime.timedelta(
days=2)
start_date = start_date.strftime("%Y-%m-%d")
end_date = DateAccessor().today().strftime("%Y-%m-%d")
msg = f"report to summarize: {str(report)}"
tracing_id = report.get(
"tracing_id", report.get("manifest_uuid", "no-tracing-id"))
LOG.info(log_json(tracing_id, msg))
update_summary_tables.s(
report.get("schema_name"),
report.get("provider_type"),
report.get("provider_uuid"),
start_date=start_date,
end_date=end_date,
manifest_id=report.get("manifest_id"),
queue_name=queue_name,
tracing_id=tracing_id,
).apply_async(queue=queue_name or UPDATE_SUMMARY_TABLES_QUEUE)
def test_get_report_context_for_date_should_download(
self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that data is returned on the reports to process."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service="iam", generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name,
provider_uuid=self.aws_provider_uuid,
)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = "1234"
compression = downloader.report.get("Compression")
report_keys = ["file1", "file2"]
mock_manifest.return_value = (
"",
{
"assemblyId": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"billingPeriod": {
"start": start_str
},
},
)
mock_check.return_value = True
expected = {
"manifest_id": None,
"assembly_id": assembly_id,
"compression": compression,
"files": report_keys
}
result = downloader.get_report_context_for_date(current_month)
with ReportManifestDBAccessor() as manifest_accessor:
manifest_entry = manifest_accessor.get_manifest(
assembly_id, self.aws_provider_uuid)
expected["manifest_id"] = manifest_entry.id
self.assertIsInstance(result, dict)
for key, value in result.items():
self.assertIn(key, expected)
self.assertEqual(value, expected.get(key))
def test_get_report_context_for_date_should_download(
self, mock_session, mock_manifest, mock_delete, mock_check):
"""Test that data is returned on the reports to process."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
auth_credential = fake_arn(service='iam', generate_account_id=True)
downloader = AWSReportDownloader(
self.mock_task,
self.fake_customer_name,
auth_credential,
self.fake_bucket_name,
provider_uuid=self.aws_provider_uuid,
)
start_str = current_month.strftime(downloader.manifest_date_format)
assembly_id = '1234'
compression = downloader.report.get('Compression')
report_keys = ['file1', 'file2']
mock_manifest.return_value = (
'',
{
'assemblyId': assembly_id,
'Compression': compression,
'reportKeys': report_keys,
'billingPeriod': {
'start': start_str
},
},
)
mock_check.return_value = True
expected = {
'manifest_id': None,
'assembly_id': assembly_id,
'compression': compression,
'files': report_keys,
}
result = downloader.get_report_context_for_date(current_month)
with ReportManifestDBAccessor() as manifest_accessor:
manifest_entry = manifest_accessor.get_manifest(
assembly_id, self.aws_provider_uuid)
expected['manifest_id'] = manifest_entry.id
self.assertIsInstance(result, dict)
for key, value in result.items():
self.assertIn(key, expected)
self.assertEqual(value, expected.get(key))