def test_update_summary_tables_without_manifest(self, mock_daily, mock_summary): """Test that summary tables are properly run without a manifest.""" self.updater = AWSReportSummaryUpdater('acct10001', self.provider, None) start_date = DateAccessor().today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() with schema_context(self.schema): bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0] bill.summary_data_updated_datetime = start_date self.accessor.commit() start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') expected_start_date = start_date.strftime('%Y-%m-%d') expected_end_date = end_date.strftime('%Y-%m-%d') self.updater.update_daily_tables(start_date_str, end_date_str) mock_daily.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)]) mock_summary.assert_not_called() self.updater.update_summary_tables(start_date_str, end_date_str) mock_summary.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)]) with AWSReportDBAccessor('acct10001', self.column_map) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.summary_data_creation_datetime) self.assertGreater(bill.summary_data_updated_datetime, start_date)
def summarize_reports(reports_to_summarize): """ Summarize reports returned from line summary task. Args: reports_to_summarize (list) list of reports to process Returns: None """ for report in reports_to_summarize: # For day-to-day summarization we choose a small window to # cover new data from a window of days. # This saves us from re-summarizing unchanged data and cuts down # on processing time. There are override mechanisms in the # Updater classes for when full-month summarization is # required. start_date = DateAccessor().today() - datetime.timedelta(days=2) start_date = start_date.strftime("%Y-%m-%d") end_date = DateAccessor().today().strftime("%Y-%m-%d") LOG.info("report to summarize: %s", str(report)) update_summary_tables.delay( report.get("schema_name"), report.get("provider_type"), report.get("provider_uuid"), start_date=start_date, end_date=end_date, manifest_id=report.get("manifest_id"), )
def test_get_manifest_context_for_date(self, mock_manifest): """Test that the manifest is read.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) start_str = current_month.strftime( self.downloader.manifest_date_format) assembly_id = "1234" compression = UNCOMPRESSED report_keys = ["file1", "file2"] mock_manifest.return_value = ( { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": { "start": start_str }, }, DateAccessor().today(), ) result = self.downloader.get_manifest_context_for_date(current_month) self.assertEqual(result.get("assembly_id"), assembly_id) self.assertEqual(result.get("compression"), compression) self.assertIsNotNone(result.get("files"))
def setUpClass(cls): """Set up the test class.""" super().setUpClass() today = DateAccessor().today_with_timezone("UTC") cls.today = today.strftime("%Y-%m-%d") cls.tomorrow = (today + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
def test_get_report_context_for_date_should_not_download( self, mock_session, mock_manifest, mock_delete, mock_check ): """Test that no data is returned when we don't want to process.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name ) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = "1234" compression = downloader.report.get("Compression") report_keys = ["file1", "file2"] mock_manifest.return_value = ( "", { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": {"start": start_str}, }, ) mock_check.return_value = False expected = {} result = downloader.get_report_context_for_date(current_month) self.assertEqual(result, expected)
def test_get_manifest_context_for_date(self, mock_session, mock_manifest, mock_delete): """Test that the manifest is read.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) downloader = AWSReportDownloader(self.fake_customer_name, self.credentials, self.data_source, provider_uuid=self.aws_provider_uuid) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = "1234" compression = downloader.report.get("Compression") report_keys = ["file1", "file2"] mock_manifest.return_value = ( "", { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": { "start": start_str }, }, DateAccessor().today(), ) result = downloader.get_manifest_context_for_date(current_month) self.assertEqual(result.get("assembly_id"), assembly_id) self.assertEqual(result.get("compression"), compression) self.assertIsNotNone(result.get("files"))
def test_get_report_context_for_date_should_not_download( self, mock_session, mock_manifest, mock_delete, mock_check): """Test that no data is returned when we don't want to process.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service='iam', generate_account_id=True) downloader = AWSReportDownloader(self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = '1234' compression = downloader.report.get('Compression') report_keys = ['file1', 'file2'] mock_manifest.return_value = ('', { 'assemblyId': assembly_id, 'Compression': compression, 'reportKeys': report_keys, 'billingPeriod': { 'start': start_str } }) mock_check.return_value = False expected = {} result = downloader.get_report_context_for_date(current_month) self.assertEqual(result, expected)
def setUpClass(cls): """Set up the test class.""" super().setUpClass() cls.schema = 'acct10001' today = DateAccessor().today_with_timezone('UTC') cls.today = today.strftime('%Y-%m-%d') cls.tomorrow = (today + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
def test_update_summary_tables_without_manifest( self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary ): """Test that summary tables are properly run without a manifest.""" # Create an updater that doesn't have a manifest updater = OCPReportSummaryUpdater(self.schema, self.provider, None) start_date = DateAccessor().today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() with schema_context(self.schema): period = self.accessor.get_usage_periods_by_date(bill_date)[0] period.summary_data_updated_datetime = start_date period.save() start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') expected_start_date = start_date.strftime('%Y-%m-%d') expected_end_date = end_date.strftime('%Y-%m-%d') updater.update_daily_tables(start_date_str, end_date_str) mock_daily.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) mock_storage_daily.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) mock_sum.assert_not_called() mock_storage_summary.assert_not_called() updater.update_summary_tables(start_date_str, end_date_str) mock_sum.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) mock_storage_summary.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) with OCPReportDBAccessor(self.schema, self.column_map) as accessor: period = accessor.get_usage_periods_by_date(bill_date)[0] self.assertIsNotNone(period.summary_data_creation_datetime) self.assertGreater(period.summary_data_updated_datetime, start_date)
def summarize_reports(reports_to_summarize, queue_name=None): """ Summarize reports returned from line summary task. Args: reports_to_summarize (list) list of reports to process Returns: None """ reports_to_summarize = [ report for report in reports_to_summarize if report ] reports_deduplicated = [ dict(t) for t in {tuple(d.items()) for d in reports_to_summarize} ] for report in reports_deduplicated: # For day-to-day summarization we choose a small window to # cover new data from a window of days. # This saves us from re-summarizing unchanged data and cuts down # on processing time. There are override mechanisms in the # Updater classes for when full-month summarization is # required. with ReportManifestDBAccessor() as manifest_accesor: if manifest_accesor.manifest_ready_for_summary( report.get("manifest_id")): if report.get("start") and report.get("end"): LOG.info("using start and end dates from the manifest") start_date = parser.parse( report.get("start")).strftime("%Y-%m-%d") end_date = parser.parse( report.get("end")).strftime("%Y-%m-%d") else: LOG.info("generating start and end dates for manifest") start_date = DateAccessor().today() - datetime.timedelta( days=2) start_date = start_date.strftime("%Y-%m-%d") end_date = DateAccessor().today().strftime("%Y-%m-%d") msg = f"report to summarize: {str(report)}" tracing_id = report.get( "tracing_id", report.get("manifest_uuid", "no-tracing-id")) LOG.info(log_json(tracing_id, msg)) update_summary_tables.s( report.get("schema_name"), report.get("provider_type"), report.get("provider_uuid"), start_date=start_date, end_date=end_date, manifest_id=report.get("manifest_id"), queue_name=queue_name, tracing_id=tracing_id, ).apply_async(queue=queue_name or UPDATE_SUMMARY_TABLES_QUEUE)
def test_get_report_context_for_date_should_download( self, mock_session, mock_manifest, mock_delete, mock_check): """Test that data is returned on the reports to process.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service="iam", generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name, provider_uuid=self.aws_provider_uuid, ) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = "1234" compression = downloader.report.get("Compression") report_keys = ["file1", "file2"] mock_manifest.return_value = ( "", { "assemblyId": assembly_id, "Compression": compression, "reportKeys": report_keys, "billingPeriod": { "start": start_str }, }, ) mock_check.return_value = True expected = { "manifest_id": None, "assembly_id": assembly_id, "compression": compression, "files": report_keys } result = downloader.get_report_context_for_date(current_month) with ReportManifestDBAccessor() as manifest_accessor: manifest_entry = manifest_accessor.get_manifest( assembly_id, self.aws_provider_uuid) expected["manifest_id"] = manifest_entry.id self.assertIsInstance(result, dict) for key, value in result.items(): self.assertIn(key, expected) self.assertEqual(value, expected.get(key))
def test_get_report_context_for_date_should_download( self, mock_session, mock_manifest, mock_delete, mock_check): """Test that data is returned on the reports to process.""" current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1) auth_credential = fake_arn(service='iam', generate_account_id=True) downloader = AWSReportDownloader( self.mock_task, self.fake_customer_name, auth_credential, self.fake_bucket_name, provider_uuid=self.aws_provider_uuid, ) start_str = current_month.strftime(downloader.manifest_date_format) assembly_id = '1234' compression = downloader.report.get('Compression') report_keys = ['file1', 'file2'] mock_manifest.return_value = ( '', { 'assemblyId': assembly_id, 'Compression': compression, 'reportKeys': report_keys, 'billingPeriod': { 'start': start_str }, }, ) mock_check.return_value = True expected = { 'manifest_id': None, 'assembly_id': assembly_id, 'compression': compression, 'files': report_keys, } result = downloader.get_report_context_for_date(current_month) with ReportManifestDBAccessor() as manifest_accessor: manifest_entry = manifest_accessor.get_manifest( assembly_id, self.aws_provider_uuid) expected['manifest_id'] = manifest_entry.id self.assertIsInstance(result, dict) for key, value in result.items(): self.assertIn(key, expected) self.assertEqual(value, expected.get(key))