def test_update_summary_tables_azure(self, mock_markup, mock_refresh): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} mock_markup.return_value = markup with ProviderDBAccessor(self.ocp_provider_uuid) as accessor: accessor.set_infrastructure(self.azure_provider_uuid, Provider.PROVIDER_AZURE) self._generate_ocp_on_azure_data() start_date = self.date_accessor.today_with_timezone("UTC") end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor( self.azure_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) initial_count = query.count() updater.update_summary_tables(start_date_str, end_date_str) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: query = azure_accessor._get_db_obj_query(summary_table_name) self.assertNotEqual(query.count(), initial_count) markup_cost = query.aggregate( Sum("markup_cost"))["markup_cost__sum"] pretax_cost = query.aggregate( Sum("pretax_cost"))["pretax_cost__sum"] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get("value") / 100), places=9) daily_summary_table_name = OCP_REPORT_TABLE_MAP[ "line_item_daily_summary"] with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: query = ocp_accessor._get_db_obj_query(daily_summary_table_name) infra_cost = query.aggregate(Sum("infra_cost"))["infra_cost__sum"] project_infra_cost = query.aggregate( Sum("project_infra_cost"))["project_infra_cost__sum"] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0)) mock_refresh.assert_called()
def setUp(self): """Set up each test.""" super().setUp() self.assembly_id = '1234' self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_id=self.azure_provider_id, ) billing_start = self.date_accessor.today_with_timezone('UTC').replace( year=2018, month=6, day=1, hour=0, minute=0, second=0) self.assembly_id = '1234' self.manifest_dict = { 'assembly_id': self.assembly_id, 'billing_period_start_datetime': billing_start, 'num_total_files': 1, 'provider_id': self.azure_provider.id, } self.accessor = AzureReportDBAccessor(self.schema, self.column_map) self.report_schema = self.accessor.report_schema self.manifest = self.manifest_accessor.add(**self.manifest_dict) self.manifest_accessor.commit()
def setUp(self): """Set up each test.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f'{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv' shutil.copy2(self.test_report_path, self.test_report) self.assembly_id = '1234' self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) billing_start = self.date_accessor.today_with_timezone('UTC').replace( year=2018, month=6, day=1, hour=0, minute=0, second=0 ) self.assembly_id = '1234' self.manifest_dict = { 'assembly_id': self.assembly_id, 'billing_period_start_datetime': billing_start, 'num_total_files': 1, 'provider_uuid': self.azure_provider_uuid, } self.accessor = AzureReportDBAccessor(self.schema, self.column_map) self.report_schema = self.accessor.report_schema self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def test_azure_update_summary_tables_with_manifest(self, mock_summary): """Test that summary tables are properly run.""" self.manifest.num_processed_files = self.manifest.num_total_files start_date = self.date_accessor.today_with_timezone("UTC") end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() with AzureReportDBAccessor(self.schema, self.column_map) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] bill.summary_data_creation_datetime = start_date bill.save() start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") expected_start_date = start_date.date() expected_end_date = end_date.date() self.assertIsNone(bill.summary_data_updated_datetime) self.updater.update_daily_tables(start_date_str, end_date_str) mock_summary.assert_not_called() self.updater.update_summary_tables(start_date_str, end_date_str) mock_summary.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)]) with AzureReportDBAccessor(self.schema, self.column_map) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.summary_data_creation_datetime) self.assertIsNotNone(bill.summary_data_updated_datetime)
def test_update_summary_tables_azure(self, mock_markup): """Test that summary tables are updated correctly.""" markup = {'value': 10, 'unit': 'percent'} mock_markup.return_value = markup with ProviderDBAccessor(self.ocp_provider_uuid) as accessor: accessor.set_infrastructure(self.azure_provider_uuid, 'AZURE') self._generate_ocp_on_azure_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.azure_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ 'ocp_on_azure_daily_summary'] query = azure_accessor._get_db_obj_query(summary_table_name) initial_count = query.count() updater.update_summary_tables(start_date_str, end_date_str) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: query = azure_accessor._get_db_obj_query(summary_table_name) self.assertNotEqual(query.count(), initial_count) markup_cost = query.aggregate( Sum('markup_cost'))['markup_cost__sum'] pretax_cost = query.aggregate( Sum('pretax_cost'))['pretax_cost__sum'] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get('value') / 100), places=9) daily_summary_table_name = OCP_REPORT_TABLE_MAP[ 'line_item_daily_summary'] with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: query = ocp_accessor._get_db_obj_query(daily_summary_table_name) infra_cost = query.aggregate(Sum('infra_cost'))['infra_cost__sum'] project_infra_cost = query.aggregate( Sum('project_infra_cost'))['project_infra_cost__sum'] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on Azure.""" cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) azure_bills = azure_get_bills_from_provider( azure_provider_uuid, self._schema, datetime.datetime.strptime(start_date, '%Y-%m-%d'), datetime.datetime.strptime(end_date, '%Y-%m-%d')) azure_bill_ids = [] with schema_context(self._schema): azure_bill_ids = [str(bill.id) for bill in azure_bills] with CostModelDBAccessor(self._schema, azure_provider_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() markup_value = Decimal(markup.get('value', 0)) / 100 # OpenShift on Azure with AzureReportDBAccessor(self._schema, self._column_map) as accessor: LOG.info( 'Updating OpenShift on Azure summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s' '\n\tCluster ID: %s, Azure Bill IDs: %s', self._schema, self._provider.uuid, start_date, end_date, cluster_id, str(azure_bill_ids)) accessor.populate_ocp_on_azure_cost_daily_summary( start_date, end_date, cluster_id, azure_bill_ids) accessor.populate_ocp_on_azure_markup_cost(markup_value, azure_bill_ids) with OCPReportDBAccessor(self._schema, self._column_map) as accessor: # This call just sends the infrastructure cost to the # OCP usage daily summary table accessor.update_summary_infrastructure_cost( cluster_id, start_date, end_date)
def update_summary_cost_model_costs(self, start_date=None, end_date=None): """Update the Azure summary table with the charge information. Args: start_date (str, Optional) - Start date of range to update derived cost. end_date (str, Optional) - End date of range to update derived cost. Returns None """ LOG.debug( "Starting charge calculation updates for provider: %s. Dates: %s-%s", self._provider.uuid, str(start_date), str(end_date), ) self._update_markup_cost(start_date, end_date) with AzureReportDBAccessor(self._schema) as accessor: LOG.debug( "Updating Azure derived cost summary for schema: %s and provider: %s", self._schema, self._provider.uuid, ) bills = accessor.bills_for_provider_uuid(self._provider.uuid, start_date) with schema_context(self._schema): for bill in bills: bill.derived_cost_datetime = DateAccessor( ).today_with_timezone("UTC") bill.save()
def create_azure_cost_entry_line_item(self, bill, product, meter, usage_date_time=None): """Create an Azure cost entry line item database object for test.""" table_name = AZURE_REPORT_TABLE_MAP['line_item'] data = self.create_columns_for_table(table_name) random_usage_date_time = bill.billing_period_start + relativedelta.relativedelta( days=random.randint(1, 15)) extra_data = { 'cost_entry_bill_id': bill.id, 'cost_entry_product_id': product.id, 'meter_id': meter.id, 'usage_date_time': usage_date_time if usage_date_time else random_usage_date_time, 'tags': { 'environment': random.choice(['dev', 'qa', 'prod']), self.fake.pystr()[:8]: self.fake.pystr()[:8], } } data.update(extra_data) with AzureReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def test_azure_update_summary_tables_new_bill(self, mock_summary): """Test that summary tables are run for a full month.""" self.manifest.num_processed_files = self.manifest.num_total_files start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date bill_date = start_date.replace(day=1).date() with schema_context(self.schema): bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0] last_day_of_month = calendar.monthrange(bill_date.year, bill_date.month)[1] start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') expected_start_date = start_date.replace(day=1).strftime('%Y-%m-%d') expected_end_date = end_date.replace(day=last_day_of_month).strftime('%Y-%m-%d') self.assertIsNone(bill.summary_data_creation_datetime) self.assertIsNone(bill.summary_data_updated_datetime) self.updater.update_daily_tables(start_date_str, end_date_str) mock_summary.assert_not_called() self.updater.update_summary_tables(start_date_str, end_date_str) mock_summary.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)]) with AzureReportDBAccessor(self.schema, self.column_map) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.summary_data_creation_datetime) self.assertIsNotNone(bill.summary_data_updated_datetime)
def create_azure_cost_entry_line_item(self, bill, product, meter, usage_date=None): """Create an Azure cost entry line item database object for test.""" table_name = AZURE_REPORT_TABLE_MAP["line_item"] data = self.create_columns_for_table(table_name) if usage_date: usage_date = usage_date.date() if isinstance( usage_date, datetime.datetime) else usage_date else: usage_date = (bill.billing_period_start + relativedelta.relativedelta( days=random.randint(1, 15))).date() extra_data = { "cost_entry_bill_id": bill.id, "cost_entry_product_id": product.id, "meter_id": meter.id, "usage_date": usage_date, "tags": { "environment": random.choice(["dev", "qa", "prod"]), self.fake.pystr()[:8]: self.fake.pystr()[:8], }, } data.update(extra_data) with AzureReportDBAccessor(self.schema) as accessor: return accessor.create_db_object(table_name, data)
def _get_sql_inputs(self, start_date, end_date): """Get the required inputs for running summary SQL.""" with AzureReportDBAccessor(self._schema) as accessor: # This is the normal processing route if self._manifest: # Override the bill date to correspond with the manifest bill_date = self._manifest.billing_period_start_datetime.date() bills = accessor.get_cost_entry_bills_query_by_provider( self._provider.uuid) bills = bills.filter(billing_period_start=bill_date).all() first_bill = bills.filter( billing_period_start=bill_date).first() do_month_update = False with schema_context(self._schema): if first_bill: do_month_update = self._determine_if_full_summary_update_needed( first_bill) if do_month_update: last_day_of_month = calendar.monthrange( bill_date.year, bill_date.month)[1] start_date = bill_date.strftime("%Y-%m-%d") end_date = bill_date.replace(day=last_day_of_month) end_date = end_date.strftime("%Y-%m-%d") LOG.info( "Overriding start and end date to process full month.") return start_date, end_date
def test_update_daily_summary_tables(self, mock_presto, mock_tag_update, mock_delete): """Test that we run Presto summary.""" start_str = self.dh.this_month_start.isoformat() end_str = self.dh.this_month_end.isoformat() start, end = self.updater._get_sql_inputs(start_str, end_str) for s, e in date_range_pair(start, end, step=settings.TRINO_DATE_STEP): expected_start, expected_end = s, e with AzureReportDBAccessor(self.schema) as accessor: with schema_context(self.schema): bills = accessor.bills_for_provider_uuid(self.azure_provider.uuid, start) bill_ids = [str(bill.id) for bill in bills] current_bill_id = bills.first().id if bills else None with CostModelDBAccessor(self.schema, self.azure_provider.uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = float(markup.get("value", 0)) / 100 start_return, end_return = self.updater.update_summary_tables(start, end) mock_delete.assert_called_with(self.azure_provider.uuid, expected_start, expected_end) mock_presto.assert_called_with( expected_start, expected_end, self.azure_provider.uuid, current_bill_id, markup_value ) mock_tag_update.assert_called_with(bill_ids, start, end) self.assertEqual(start_return, start) self.assertEqual(end_return, end)
def create_azure_meter(self): """Create an Azure meter database object for test.""" table_name = AZURE_REPORT_TABLE_MAP['meter'] data = self.create_columns_for_table(table_name) with AzureReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def db_accessor(self): """Return the accessor for the infrastructure provider.""" if self.provider_type == Provider.PROVIDER_AWS: return AWSReportDBAccessor(self.schema_name) elif self.provider_type == Provider.PROVIDER_AZURE: return AzureReportDBAccessor(self.schema_name) return None
def test_refresh_openshift_on_infrastructure_views(self): """Test that the combined OpenShift views are refreshed.""" self._generate_ocp_on_aws_data(cluster_id="my-ocp-cluster-1") self._generate_ocp_on_azure_data(cluster_id="my-ocp-cluster-2") start_date = self.date_accessor.today_with_timezone("UTC") end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor( self.azure_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_count = query.count() summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_project_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_project_count = query.count() with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_count = query.count() summary_table_name = AWS_CUR_TABLE_MAP[ "ocp_on_aws_project_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_project_count = query.count() updater.refresh_openshift_on_infrastructure_views() with schema_context(self.schema): all_count = OCPAllCostLineItemDailySummary.objects.count() all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count( ) self.assertEqual(all_count, azure_count + aws_count) self.assertEqual(all_project_count, azure_project_count + aws_project_count)
def create_azure_meter(self, provider_uuid): """Create an Azure meter database object for test.""" table_name = AZURE_REPORT_TABLE_MAP["meter"] data = self.create_columns_for_table(table_name) data["provider_id"] = provider_uuid with AzureReportDBAccessor(self.schema) as accessor: return accessor.create_db_object(table_name, data)
def create_azure_cost_entry_product(self, provider_uuid): """Create an Azure cost entry product database object for test.""" table_name = AZURE_REPORT_TABLE_MAP['product'] data = self.create_columns_for_table(table_name) data['provider_id'] = provider_uuid with AzureReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def test_refresh_openshift_on_infrastructure_views(self, mock_view_refresh): """Test that the combined OpenShift views are refreshed.""" start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.azure_provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_view_refresh.assert_called_with( OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) mock_view_refresh.reset_mock() updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.aws_provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) expected_calls = [ call(OCP_ON_AWS_MATERIALIZED_VIEWS), call(OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) ] mock_view_refresh.assert_has_calls(expected_calls) with AzureReportDBAccessor(self.schema) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_count = query.count() summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_project_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_project_count = query.count() with AWSReportDBAccessor(self.schema) as aws_accessor: summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_count = query.count() summary_table_name = AWS_CUR_TABLE_MAP[ "ocp_on_aws_project_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_project_count = query.count() updater.refresh_openshift_on_infrastructure_views( OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) with schema_context(self.schema): all_count = OCPAllCostLineItemDailySummary.objects.count() all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count( ) self.assertEqual(all_count, azure_count + aws_count) self.assertEqual(all_project_count, azure_project_count + aws_project_count)
def test_update_summary_tables_azure( self, mock_cost_model # , mock_ocpall_proj_summ, mock_ocpall_summ, mock_ocpall_persp ): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} mock_cost_model.markup = markup start_date = self.dh.this_month_start end_date = self.dh.this_month_end updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.azure_provider, manifest=None) updater.update_summary_tables( start_date, end_date, self.ocp_on_azure_ocp_provider.uuid, self.azure_provider.uuid, Provider.PROVIDER_AZURE, ) summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] with AzureReportDBAccessor(self.schema) as azure_accessor: query = azure_accessor._get_db_obj_query( summary_table_name).filter( cost_entry_bill__billing_period_start=start_date) markup_cost = query.aggregate( Sum("markup_cost"))["markup_cost__sum"] pretax_cost = query.aggregate( Sum("pretax_cost"))["pretax_cost__sum"] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get("value") / 100), places=5) daily_summary_table_name = OCP_REPORT_TABLE_MAP[ "line_item_daily_summary"] with OCPReportDBAccessor(self.schema) as ocp_accessor: query = ocp_accessor._get_db_obj_query( daily_summary_table_name).filter( report_period__provider=self.ocp_on_azure_ocp_provider, report_period__report_period_start=self.dh. this_month_start, ) infra_cost = query.aggregate( Sum("infrastructure_raw_cost"))["infrastructure_raw_cost__sum"] project_infra_cost = query.aggregate( Sum("infrastructure_project_raw_cost") )["infrastructure_project_raw_cost__sum"] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def purge_expired_report_data(self, expired_date=None, provider_uuid=None, simulate=False): """Remove report data with a billing start period before specified date. Args: expired_date (datetime.datetime): The cutoff date for removing data. provider_uuid (uuid): The DB id of the provider to purge data for. simulate (bool): Whether to simulate the removal. Returns: ([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start' """ LOG.info("Calling purge_expired_report_data for azure") with AzureReportDBAccessor(self._schema) as accessor: if (expired_date is None and provider_uuid is None) or ( # noqa: W504 expired_date is not None and provider_uuid is not None ): err = "This method must be called with either expired_date or provider_uuid" raise AzureReportDBCleanerError(err) removed_items = [] if expired_date is not None: bill_objects = accessor.get_bill_query_before_date(expired_date) else: bill_objects = accessor.get_cost_entry_bills_query_by_provider(provider_uuid) with schema_context(self._schema): for bill in bill_objects.all(): bill_id = bill.id removed_provider_uuid = bill.provider_id removed_billing_period_start = bill.billing_period_start if not simulate: lineitem_query = accessor.get_lineitem_query_for_billid(bill_id) del_count, remainder = mini_transaction_delete(lineitem_query) LOG.info("Removing %s cost entry line items for bill id %s", del_count, bill_id) summary_query = accessor.get_summary_query_for_billid(bill_id) del_count, remainder = mini_transaction_delete(summary_query) LOG.info("Removing %s cost entry summary items for bill id %s", del_count, bill_id) LOG.info( "Report data removed for Account Payer ID: %s with billing period: %s", removed_provider_uuid, removed_billing_period_start, ) removed_items.append( { "provider_uuid": removed_provider_uuid, "billing_period_start": str(removed_billing_period_start), } ) if not simulate: bill_objects.delete() return removed_items
def process(self): """Process cost/usage file. Returns: (None) """ row_count = 0 is_full_month = self._should_process_full_month() self._delete_line_items(AzureReportDBAccessor) opener, mode = self._get_file_opener(self._compression) with opener(self._report_path, mode, encoding="utf-8-sig") as f: header = normalize_header(f.readline()) with AzureReportDBAccessor(self._schema) as report_db: temp_table = report_db.create_temp_table( self.table_name._meta.db_table, drop_column="id") LOG.info("File %s opened for processing", str(f)) reader = csv.DictReader(f, fieldnames=header) for row in reader: if not self._should_process_row(row, "UsageDateTime", is_full_month): continue self.create_cost_entry_objects(row, report_db) if len(self.processed_report.line_items ) >= self._batch_size: LOG.info( "Saving report rows %d to %d for %s", row_count, row_count + len(self.processed_report.line_items), self._report_name, ) self._save_to_db(temp_table, report_db) row_count += len(self.processed_report.line_items) self._update_mappings() if self.processed_report.line_items: LOG.info( "Saving report rows %d to %d for %s", row_count, row_count + len(self.processed_report.line_items), self._report_name, ) self._save_to_db(temp_table, report_db) row_count += len(self.processed_report.line_items) if self.processed_report.line_items: report_db.merge_temp_table(self.table_name._meta.db_table, temp_table, self.line_item_columns) LOG.info( "Completed report processing for file: %s and schema: %s", self._report_name, self._schema) if not settings.DEVELOPMENT: LOG.info("Removing processed file: %s", self._report_path) remove(self._report_path) return True
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str) A start date and end date. """ start_date, end_date = self._get_sql_inputs(start_date, end_date) with CostModelDBAccessor(self._schema, self._provider.uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = float(markup.get("value", 0)) / 100 with AzureReportDBAccessor(self._schema) as accessor: # Need these bills on the session to update dates after processing with schema_context(self._schema): bills = accessor.bills_for_provider_uuid( self._provider.uuid, start_date) bill_ids = [str(bill.id) for bill in bills] current_bill_id = bills.first().id if bills else None for start, end in date_range_pair(start_date, end_date, step=settings.TRINO_DATE_STEP): LOG.info( "Updating Azure report summary tables via Presto: \n\tSchema: %s" "\n\tProvider: %s \n\tDates: %s - %s", self._schema, self._provider.uuid, start, end, ) accessor.delete_line_item_daily_summary_entries_for_date_range( self._provider.uuid, start, end) accessor.populate_line_item_daily_summary_table_presto( start, end, self._provider.uuid, current_bill_id, markup_value) accessor.populate_enabled_tag_keys(start, end, bill_ids) accessor.populate_tags_summary_table(bill_ids, start_date, end_date) accessor.update_line_item_daily_summary_with_enabled_tags( start_date, end_date, bill_ids) for bill in bills: if bill.summary_data_creation_datetime is None: bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone( "UTC") bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone( "UTC") bill.save() return start_date, end_date
def purge_expired_report_data_by_date(self, expired_date, simulate=False): partition_from = str(date(expired_date.year, expired_date.month, 1)) with AzureReportDBAccessor(self._schema) as accessor: all_bill_objects = accessor.get_bill_query_before_date( expired_date).all() table_names = [ accessor._table_map["ocp_on_azure_daily_summary"], accessor._table_map["ocp_on_azure_project_daily_summary"], accessor.line_item_daily_summary_table._meta.db_table, accessor.ocpall_line_item_daily_summary_table._meta.db_table, accessor.ocpall_line_item_project_daily_summary_table._meta. db_table, ] table_names.extend(UI_SUMMARY_TABLES) table_models = [get_model(tn) for tn in table_names] with schema_context(self._schema): removed_items = [] all_providers = set() all_period_starts = set() # Iterate over the remainder as they could involve much larger amounts of data for bill in all_bill_objects: removed_items.append({ "provider_uuid": bill.provider_id, "billing_period_start": str(bill.billing_period_start) }) all_providers.add(bill.provider_id) all_period_starts.add(str(bill.billing_period_start)) LOG.info( f"Deleting data for providers {all_providers} and periods {all_period_starts}" ) if not simulate: # Will call trigger to detach, truncate, and drop partitions LOG.info( "Deleting table partitions total for the following tables: " + f"{table_names} with partitions <= {partition_from}") del_count = execute_delete_sql( PartitionedTable.objects.filter( schema_name=self._schema, partition_of_table_name__in=table_names, partition_parameters__default=False, partition_parameters__from__lte=partition_from, )) LOG.info(f"Deleted {del_count} table partitions") if not simulate: cascade_delete(all_bill_objects.query.model, all_bill_objects, skip_relations=table_models) return removed_items
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.accessor = AzureReportDBAccessor("acct10001") cls.report_schema = cls.accessor.report_schema cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.schema) cls.date_accessor = DateHelper() cls.manifest_accessor = ReportManifestDBAccessor()
def create_azure_cost_entry_product(self, provider_uuid, instance_id=None): """Create an Azure cost entry product database object for test.""" table_name = AZURE_REPORT_TABLE_MAP["product"] data = self.create_columns_for_table(table_name) data["provider_id"] = provider_uuid if instance_id: data["instance_id"] = instance_id with AzureReportDBAccessor(self.schema) as accessor: return accessor.create_db_object(table_name, data)
def test_azure_update_summary_charge_info(self): """Test to verify Azure derived cost summary is calculated.""" start_date = self.date_accessor.today_with_timezone('UTC') bill_date = start_date.replace(day=1).date() self.updater.update_summary_charge_info() with AzureReportDBAccessor(self.schema, self.column_map) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.derived_cost_datetime)
def purge_expired_report_data(self, expired_date=None, provider_uuid=None, simulate=False): """Remove report data with a billing start period before specified date. Args: expired_date (datetime.datetime): The cutoff date for removing data. provider_uuid (uuid): The DB id of the provider to purge data for. simulate (bool): Whether to simulate the removal. Returns: ([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start' """ LOG.info("Calling purge_expired_report_data for azure") with AzureReportDBAccessor(self._schema) as accessor: if (expired_date is None and provider_uuid is None) or ( # noqa: W504 expired_date is not None and provider_uuid is not None): err = "This method must be called with either expired_date or provider_uuid" raise AzureReportDBCleanerError(err) removed_items = [] all_providers = set() all_period_starts = set() if expired_date is not None: return self.purge_expired_report_data_by_date( expired_date, simulate=simulate) bill_objects = accessor.get_cost_entry_bills_query_by_provider( provider_uuid) with schema_context(self._schema): for bill in bill_objects.all(): removed_items.append({ "provider_uuid": bill.provider_id, "billing_period_start": str(bill.billing_period_start) }) all_providers.add(bill.provider_id) all_period_starts.add(str(bill.billing_period_start)) LOG.info( f"Deleting data for providers {all_providers} and periods {all_period_starts}" ) if not simulate: cascade_delete(bill_objects.query.model, bill_objects) return removed_items
def process(self): """Process cost/usage file. Returns: (None) """ row_count = 0 is_full_month = self._should_process_full_month() self._delete_line_items(AzureReportDBAccessor, self.column_map) # pylint: disable=invalid-name opener, mode = self._get_file_opener(self._compression) with opener(self._report_path, mode, encoding="utf-8-sig") as f: with AzureReportDBAccessor(self._schema, self.column_map) as report_db: LOG.info("File %s opened for processing", str(f)) reader = csv.DictReader(f) for row in reader: if not self._should_process_row(row, "UsageDateTime", is_full_month): continue _ = self.create_cost_entry_objects(row, report_db) if len(self.processed_report.line_items ) >= self._batch_size: LOG.info( "Saving report rows %d to %d for %s", row_count, row_count + len(self.processed_report.line_items), self._report_name, ) self._save_to_db(AZURE_REPORT_TABLE_MAP["line_item"], report_db) row_count += len(self.processed_report.line_items) self._update_mappings() if self.processed_report.line_items: LOG.info( "Saving report rows %d to %d for %s", row_count, row_count + len(self.processed_report.line_items), self._report_name, ) self._save_to_db(AZURE_REPORT_TABLE_MAP["line_item"], report_db) row_count += len(self.processed_report.line_items) LOG.info( "Completed report processing for file: %s and schema: %s", self._report_name, self._schema) if not settings.DEVELOPMENT: LOG.info("Removing processed file: %s", self._report_path) remove(self._report_path) return True
def test_azure_update_summary_cost_model_costs(self): """Test to verify Azure derived cost summary is calculated.""" updater = AzureCostModelCostUpdater(schema=self.schema, provider=self.azure_provider) start_date = DateAccessor().today_with_timezone("UTC") bill_date = start_date.replace(day=1).date() updater.update_summary_cost_model_costs() with AzureReportDBAccessor(self.schema) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.derived_cost_datetime)
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = AzureReportDBAccessor("acct10001", cls.column_map) cls.report_schema = cls.accessor.report_schema cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.schema, cls.column_map) cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor()