def test_get_bill_ids_from_provider(self): """Test that bill IDs are returned for an AWS provider.""" with schema_context(self.schema): expected_bill_ids = AWSCostEntryBill.objects.values_list("id") expected_bill_ids = sorted([bill_id[0] for bill_id in expected_bill_ids]) bills = utils.get_bills_from_provider(self.aws_provider_uuid, self.schema) with schema_context(self.schema): bill_ids = sorted([bill.id for bill in bills]) self.assertEqual(bill_ids, expected_bill_ids) # Try with unknown provider uuid bills = utils.get_bills_from_provider(self.unkown_test_provider_uuid, self.schema) self.assertEqual(bills, [])
def test_get_bill_ids_from_provider_with_start_and_end_date(self): """Test that bill IDs are returned for an AWS provider with both dates.""" date_accessor = DateAccessor() with ProviderDBAccessor(provider_uuid=self.aws_test_provider_uuid ) as provider_accessor: provider = provider_accessor.get_provider() with AWSReportDBAccessor(schema=self.test_schema, column_map=self.column_map) as accessor: report_schema = accessor.report_schema creator = ReportObjectCreator(accessor, self.column_map, report_schema.column_types) end_date = date_accessor.today_with_timezone('utc').replace(day=1) start_date = end_date for i in range(2): start_date = start_date - relativedelta(months=i) print(start_date) bill = creator.create_cost_entry_bill(bill_date=start_date) bill_table_name = AWS_CUR_TABLE_MAP['bill'] bill_obj = getattr(accessor.report_schema, bill_table_name) bills = accessor.get_cost_entry_bills_query_by_provider( provider.id) bills = bills.filter(bill_obj.billing_period_start>=start_date.date())\ .filter(bill_obj.billing_period_start<=end_date.date()).all() expected_bill_ids = [str(bill.id) for bill in bills] bills = utils.get_bills_from_provider(self.aws_test_provider_uuid, self.test_schema, start_date=start_date, end_date=end_date) bill_ids = [str(bill.id) for bill in bills] self.assertEqual(bill_ids, expected_bill_ids)
def update_daily_tables(self, start_date, end_date): """Populate the daily tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str): A start date and end date. """ start_date, end_date = self._get_sql_inputs(start_date, end_date) bills = get_bills_from_provider( self._provider.uuid, self._schema, datetime.datetime.strptime(start_date, "%Y-%m-%d"), datetime.datetime.strptime(end_date, "%Y-%m-%d"), ) bill_ids = [] with schema_context(self._schema): bill_ids = [str(bill.id) for bill in bills] with AWSReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date): LOG.info( "Updating AWS report daily tables for \n\tSchema: %s" "\n\tProvider: %s \n\tDates: %s - %s", self._schema, self._provider.uuid, start, end, ) accessor.populate_line_item_daily_table(start, end, bill_ids) return start_date, end_date
def test_get_bill_ids_from_provider_with_start_and_end_date(self): """Test that bill IDs are returned for an AWS provider with both dates.""" date_accessor = DateAccessor() with ProviderDBAccessor( provider_uuid=self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() with AWSReportDBAccessor(schema=self.schema) as accessor: end_date = date_accessor.today_with_timezone("utc").replace(day=1) start_date = end_date for i in range(2): start_date = start_date - relativedelta(months=i) bills = accessor.get_cost_entry_bills_query_by_provider( provider.uuid) with schema_context(self.schema): bills = (bills.filter( billing_period_start__gte=start_date.date()).filter( billing_period_start__lte=end_date.date()).all()) expected_bill_ids = [str(bill.id) for bill in bills] bills = utils.get_bills_from_provider(self.aws_provider_uuid, self.schema, start_date=start_date, end_date=end_date) with schema_context(self.schema): bill_ids = [str(bill.id) for bill in bills] self.assertEqual(bill_ids, expected_bill_ids)
def update_daily_tables(self, start_date, end_date): """Populate the daily tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str): A start date and end date. """ start_date, end_date = self._get_sql_inputs(start_date, end_date) bills = get_bills_from_provider( self._provider.uuid, self._schema_name, start_date, end_date ) bill_ids = [str(bill.id) for bill in bills] LOG.info('Updating AWS report daily tables for \n\tSchema: %s' '\n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: accessor.populate_line_item_daily_table(start_date, end_date, bill_ids) return start_date, end_date
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ infra_map = self._get_ocp_infra_map(start_date, end_date) cluster_id = self._get_ocp_cluster_id_for_provider( self._provider, start_date, end_date, infra_map) if cluster_id: aws_uuid = self._get_aws_provider_uuid_from_map( self._provider, infra_map) aws_bills = get_bills_from_provider(aws_uuid, self._schema_name, start_date, end_date) aws_bill_ids = [str(bill.id) for bill in aws_bills] # OpenShift on AWS with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on AWS summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s' '\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema_name, self._provider.uuid, start_date, end_date, cluster_id, str(aws_bill_ids)) accessor.populate_ocp_on_aws_cost_daily_summary( start_date, end_date, cluster_id, aws_bill_ids) accessor.commit() else: LOG.info( 'Provider: %s is not part of an OCP-on-AWS configuration.', self._provider.name) # This needs to always run regardless of whether the OpenShift # cluster is tied to a cloud provider with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date) accessor.commit()
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str) A start date and end date. """ start_date, end_date = self._get_sql_inputs(start_date, end_date) bills = get_bills_from_provider( self._provider.uuid, self._schema, datetime.datetime.strptime(start_date, "%Y-%m-%d"), datetime.datetime.strptime(end_date, "%Y-%m-%d"), ) bill_ids = [] with schema_context(self._schema): bill_ids = [str(bill.id) for bill in bills] with AWSReportDBAccessor(self._schema) as accessor: # Need these bills on the session to update dates after processing bills = accessor.bills_for_provider_uuid(self._provider.uuid, start_date) for start, end in date_range_pair(start_date, end_date): LOG.info( "Updating AWS report summary tables: \n\tSchema: %s" "\n\tProvider: %s \n\tDates: %s - %s", self._schema, self._provider.uuid, start, end, ) accessor.populate_line_item_daily_summary_table( start, end, bill_ids) accessor.populate_tags_summary_table(bill_ids, start_date, end_date) for bill in bills: if bill.summary_data_creation_datetime is None: bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone( "UTC") bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone( "UTC") bill.save() return start_date, end_date
def _update_markup_cost(self, start_date, end_date): """Store markup costs.""" try: bills = get_bills_from_provider(self._provider.uuid, self._schema, start_date, end_date) with CostModelDBAccessor( self._schema, self._provider.uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = float(markup.get("value", 0)) / 100 with AWSReportDBAccessor(self._schema) as report_accessor: with schema_context(self._schema): bill_ids = [str(bill.id) for bill in bills] report_accessor.populate_markup_cost(markup_value, bill_ids) except AWSCostModelCostUpdaterError as error: LOG.error("Unable to update markup costs. Error: %s", str(error))
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ infra_map = self._get_ocp_infra_map(start_date, end_date) cluster_id = self._get_ocp_cluster_id_for_provider( self._provider, start_date, end_date, infra_map) if cluster_id: aws_uuid = self._get_aws_provider_uuid_from_map( self._provider, infra_map) aws_bills = get_bills_from_provider( aws_uuid, self._schema_name, datetime.datetime.strptime(start_date, '%Y-%m-%d'), datetime.datetime.strptime(end_date, '%Y-%m-%d')) aws_bill_ids = [] with schema_context(self._schema_name): aws_bill_ids = [str(bill.id) for bill in aws_bills] with CostModelDBAccessor(self._schema_name, aws_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() markup_value = float(markup.get('value', 0)) / 100 # OpenShift on AWS with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on AWS summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s' '\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema_name, self._provider.uuid, start_date, end_date, cluster_id, str(aws_bill_ids)) accessor.populate_ocp_on_aws_cost_daily_summary( start_date, end_date, cluster_id, aws_bill_ids) accessor.populate_ocp_on_aws_markup_cost( markup_value, aws_bill_ids) else: LOG.info( 'Provider: %s is not part of an OCP-on-AWS configuration.', self._provider.name)
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str) A start date and end date. """ start_date, end_date = self._get_sql_inputs(start_date, end_date) bills = get_bills_from_provider( self._provider.uuid, self._schema_name, datetime.datetime.strptime(start_date, '%Y-%m-%d'), datetime.datetime.strptime(end_date, '%Y-%m-%d')) bill_ids = [] with schema_context(self._schema_name): bill_ids = [str(bill.id) for bill in bills] with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: # Need these bills on the session to update dates after processing bills = accessor.bills_for_provider_id(self._provider.id, start_date) LOG.info( 'Updating AWS report summary tables: \n\tSchema: %s' '\n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_line_item_daily_summary_table( start_date, end_date, bill_ids) accessor.populate_tags_summary_table() for bill in bills: if bill.summary_data_creation_datetime is None: bill.summary_data_creation_datetime = \ self._date_accessor.today_with_timezone('UTC') bill.summary_data_updated_datetime = \ self._date_accessor.today_with_timezone('UTC') bill.save() accessor.commit() return start_date, end_date
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) aws_bills = get_bills_from_provider(self._provider.uuid, self._schema_name, start_date, end_date) aws_bill_ids = [str(bill.id) for bill in aws_bills] # OpenShift on AWS with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on AWS summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_ocp_on_aws_cost_daily_summary( start_date, end_date, cluster_id, aws_bill_ids) accessor.commit() if cluster_id: with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date) accessor.commit()
def test_get_bill_ids_from_provider(self): """Test that bill IDs are returned for an AWS provider.""" date_accessor = DateAccessor() with AWSReportDBAccessor(schema=self.test_schema, column_map=self.column_map) as accessor: report_schema = accessor.report_schema creator = ReportObjectCreator(accessor, self.column_map, report_schema.column_types) expected_bill_ids = [] end_date = date_accessor.today_with_timezone('utc').replace(day=1) start_date = end_date for i in range(2): start_date = start_date - relativedelta(months=i) bill = creator.create_cost_entry_bill(bill_date=start_date) expected_bill_ids.append(str(bill.id)) bills = utils.get_bills_from_provider(self.aws_test_provider_uuid, self.test_schema) bill_ids = [str(bill.id) for bill in bills] self.assertEqual(sorted(bill_ids), sorted(expected_bill_ids))
def test_get_bill_ids_from_provider(self): """Test that bill IDs are returned for an AWS provider.""" date_accessor = DateAccessor() creator = ReportObjectCreator(self.schema, self.column_map) expected_bill_ids = [] end_date = date_accessor.today_with_timezone("utc").replace(day=1) start_date = end_date for i in range(2): start_date = start_date - relativedelta(months=i) bill = creator.create_cost_entry_bill( provider_uuid=self.aws_provider_uuid, bill_date=start_date) with schema_context(self.schema): expected_bill_ids.append(str(bill.id)) bills = utils.get_bills_from_provider(self.aws_provider_uuid, self.schema) with schema_context(self.schema): bill_ids = [str(bill.id) for bill in bills] self.assertEqual(sorted(bill_ids), sorted(expected_bill_ids))