def create_ocpawscostlineitem_project_daily_summary(self, account_id, schema): """Create an ocpawscostlineitem_project_daily_summary object for test.""" table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_project_daily_summary"] data = self.create_columns_for_table(table_name) with AccountAliasAccessor(account_id, schema) as accessor: account_alias = accessor._get_db_obj_query().first() data = { "account_alias_id": account_alias.id, "cost_entry_bill": self.create_cost_entry_bill(str(uuid.uuid4())), "namespace": self.fake.pystr()[:8], "pod": self.fake.pystr()[:8], "node": self.fake.pystr()[:8], "usage_start": self.make_datetime_aware(self.fake.past_datetime()), "usage_end": self.make_datetime_aware(self.fake.past_datetime()), "product_code": self.fake.pystr()[:8], "usage_account_id": self.fake.pystr()[:8], } with OCPReportDBAccessor(self.schema) as accessor: return accessor.create_db_object(table_name, data)
def create_ocp_report_period(self, provider_uuid, period_date=None, cluster_id=None): """Create an OCP report database object for test.""" table_name = OCP_REPORT_TABLE_MAP['report_period'] period_start = self.make_datetime_aware(self.fake.past_datetime()).date().replace(day=1) period_end = period_start + relativedelta.relativedelta(days=random.randint(1, 15)) data = { 'cluster_id': cluster_id if cluster_id else self.fake.pystr()[:8], 'provider_id': provider_uuid, 'report_period_start': period_start, 'report_period_end': period_end, } if period_date: period_start = period_date.replace(day=1).date() period_end = period_start + relativedelta.relativedelta(months=1) data['report_period_start'] = period_start data['report_period_end'] = period_end with OCPReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def create_ocp_storage_line_item(self, report_period, report, pod=None, namespace=None): """Create an OCP storage line item database object for test.""" table_name = OCP_REPORT_TABLE_MAP['storage_line_item'] data = self.create_columns_for_table(table_name) for key in data: if 'bytes' in key or 'byte' in key: data[key] = data[key] * Decimal(pow(2, 30)) * 5 data['report_period_id'] = report_period.id data['report_id'] = report.id if pod: data['pod'] = pod if namespace: data['namespace'] = namespace with OCPReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str) A start date and end date. """ start_date, end_date = self._get_sql_inputs(start_date, end_date) LOG.info( 'Updating OpenShift report summary tables for \n\tSchema: %s ' '\n\tProvider: %s \n\tCluster: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, self._cluster_id, start_date, end_date) report_periods = None with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: report_periods = accessor.report_periods_for_provider_id( self._provider.id, start_date) accessor.populate_line_item_daily_summary_table( start_date, end_date, self._cluster_id) accessor.populate_pod_label_summary_table() accessor.populate_storage_line_item_daily_summary_table( start_date, end_date, self._cluster_id) accessor.populate_volume_claim_label_summary_table() accessor.populate_volume_label_summary_table() for period in report_periods: if period.summary_data_creation_datetime is None: period.summary_data_creation_datetime = \ self._date_accessor.today_with_timezone('UTC') period.summary_data_updated_datetime = \ self._date_accessor.today_with_timezone('UTC') period.save() return start_date, end_date
def test_update_aws_summary_tables(self, mock_utility, mock_ocp_on_aws, mock_tag_summary, mock_map): """Test that summary tables are properly run for an OCP provider.""" fake_bills = MagicMock() fake_bills.__iter__.return_value = [Mock(), Mock()] first = Mock() bill_id = 1 first.return_value.id = bill_id fake_bills.first = first mock_utility.return_value = fake_bills start_date = self.dh.today.date() end_date = start_date + datetime.timedelta(days=1) with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() with OCPReportDBAccessor(self.schema_name) as accessor: report_period = accessor.report_periods_for_provider_uuid( self.ocp_test_provider_uuid, start_date) with schema_context(self.schema_name): current_ocp_report_period_id = report_period.id mock_map.return_value = { self.ocp_test_provider_uuid: (self.aws_provider_uuid, Provider.PROVIDER_AWS) } updater = OCPCloudParquetReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) updater.update_aws_summary_tables(self.ocp_test_provider_uuid, self.aws_test_provider_uuid, start_date, end_date) mock_ocp_on_aws.assert_called_with( start_date, end_date, self.ocp_test_provider_uuid, self.aws_test_provider_uuid, current_ocp_report_period_id, bill_id, decimal.Decimal(0), )
def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on AWS.""" if isinstance(start_date, str): start_date = parser.parse(start_date) if isinstance(end_date, str): end_date = parser.parse(end_date) cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) aws_bills = aws_get_bills_from_provider(aws_provider_uuid, self._schema, start_date, end_date) aws_bill_ids = [] with schema_context(self._schema): aws_bill_ids = [str(bill.id) for bill in aws_bills] with CostModelDBAccessor(self._schema, aws_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 # OpenShift on AWS with AWSReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date): LOG.info( "Updating OpenShift on AWS summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, AWS Bill IDs: %s", self._schema, self._provider.uuid, start, end, cluster_id, str(aws_bill_ids), ) accessor.populate_ocp_on_aws_cost_daily_summary(start, end, cluster_id, aws_bill_ids) accessor.populate_ocp_on_aws_markup_cost(markup_value, aws_bill_ids) accessor.populate_ocp_on_aws_tags_summary_table() self.refresh_openshift_on_infrastructure_views(OCP_ON_AWS_MATERIALIZED_VIEWS) with OCPReportDBAccessor(self._schema) as accessor: # This call just sends the infrastructure cost to the # OCP usage daily summary table accessor.update_summary_infrastructure_cost(cluster_id, start_date, end_date)
def test_update_summary_tables_existing_period_done_processing( self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary): """Test that summary tables are not run for a full month.""" start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() period = self.accessor.get_usage_periods_by_date(bill_date)[0] period.summary_data_creation_datetime = start_date self.accessor.commit() start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') expected_start_date = start_date.strftime('%Y-%m-%d') expected_end_date = end_date.strftime('%Y-%m-%d') self.assertIsNone(period.summary_data_updated_datetime) self.updater.update_daily_tables(start_date_str, end_date_str) mock_daily.assert_called_with(expected_start_date, expected_end_date, self.report_period.cluster_id) mock_storage_daily.assert_called_with(expected_start_date, expected_end_date, self.report_period.cluster_id) mock_sum.assert_not_called() mock_storage_summary.assert_not_called() self.updater.update_summary_tables(start_date_str, end_date_str) mock_sum.assert_called_with(expected_start_date, expected_end_date, self.report_period.cluster_id) mock_storage_summary.assert_called_with(expected_start_date, expected_end_date, self.report_period.cluster_id) with OCPReportDBAccessor('acct10001', self.column_map) as accessor: period = accessor.get_usage_periods_by_date(bill_date)[0] self.assertIsNotNone(period.summary_data_creation_datetime) self.assertIsNotNone(period.summary_data_updated_datetime)
def __init__(self, schema_name, report_path, compression, provider_uuid): """Initialize base class.""" super().__init__( schema_name=schema_name, report_path=report_path, compression=compression, provider_uuid=provider_uuid, manifest_id=None, processed_report=ProcessedOCPReport(), ) self._report_name = path.basename(report_path) self._cluster_id = utils.get_cluster_id_from_provider(provider_uuid) self._datetime_format = Config.OCP_DATETIME_STR_FORMAT self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE with OCPReportDBAccessor(self._schema) as report_db: self.existing_report_periods_map = report_db.get_report_periods() self.existing_report_map = report_db.get_reports() self.line_item_columns = None
def update_cost_summary_table(self, start_date, end_date): """Populate the cost summary tables. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) # This needs to always run regardless of whether the OpenShift # cluster is tied to a cloud provider with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date)
def test_update_summary_tables_without_manifest(self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary): """Test that summary tables are properly run without a manifest.""" # Create an updater that doesn't have a manifest updater = OCPReportSummaryUpdater(self.schema, self.provider, None) start_date = DateAccessor().today_with_timezone("UTC") end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() with schema_context(self.schema): period = self.accessor.get_usage_periods_by_date(bill_date)[0] period.summary_data_updated_datetime = start_date period.save() start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") updater.update_daily_tables(start_date_str, end_date_str) mock_daily.assert_called_with(start_date.date(), end_date.date(), self.report_period.cluster_id) mock_storage_daily.assert_called_with(start_date.date(), end_date.date(), self.report_period.cluster_id) mock_sum.assert_not_called() mock_storage_summary.assert_not_called() updater.update_summary_tables(start_date_str, end_date_str) mock_sum.assert_called_with(start_date.date(), end_date.date(), self.report_period.cluster_id) mock_storage_summary.assert_called_with(start_date.date(), end_date.date(), self.report_period.cluster_id) with OCPReportDBAccessor(self.schema, self.column_map) as accessor: period = accessor.get_usage_periods_by_date(bill_date)[0] self.assertIsNotNone(period.summary_data_creation_datetime) self.assertGreater(period.summary_data_updated_datetime, self.today)
def process(self, parquet_base_filename, daily_data_frames): """Filter data and convert to parquet.""" for ocp_provider_uuid, infra_tuple in self.ocp_infrastructure_map.items(): infra_provider_uuid = infra_tuple[0] if infra_provider_uuid != self.provider_uuid: continue msg = ( f"Processing OpenShift on {self.provider_type} to parquet." f"\n\tStart date: {str(self.start_date)}\n\tFile: {str(self.report_file)}" ) LOG.info(msg) # Get OpenShift topology data with OCPReportDBAccessor(self.schema_name) as accessor: if not accessor.get_cluster_for_provider(ocp_provider_uuid): LOG.info( f"No cluster information available for OCP Provider: {ocp_provider_uuid}," + "skipping OCP on Cloud parquet processing." ) continue cluster_topology = accessor.get_openshift_topology_for_provider(ocp_provider_uuid) # Get matching tags report_period_id = self.get_report_period_id(ocp_provider_uuid) matched_tags = [] if self.has_enabled_ocp_labels: LOG.info("Getting matching tags from Postgres.") matched_tags = self.db_accessor.get_openshift_on_cloud_matched_tags(self.bill_id, report_period_id) if not matched_tags: LOG.info("Matched tags not yet available via Postgres. Getting matching tags from Trino.") matched_tags = self.db_accessor.get_openshift_on_cloud_matched_tags_trino( self.provider_uuid, ocp_provider_uuid, self.start_date, self.end_date ) for i, daily_data_frame in enumerate(daily_data_frames): openshift_filtered_data_frame = self.ocp_on_cloud_data_processor( daily_data_frame, cluster_topology, matched_tags ) self.create_ocp_on_cloud_parquet( openshift_filtered_data_frame, parquet_base_filename, i, ocp_provider_uuid )
def setUpClass(cls): """Set up the test class with required objects.""" # These test reports should be replaced with OCP reports once processor is impelmented. cls.test_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv' cls.storage_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv' cls.unknown_report = './tests/data/test_cur.csv' cls.test_report_gzip = './tests/data/test_cur.csv.gz' cls.provider_id = 1 cls.ocp_processor = OCPReportProcessor(schema_name='acct10001', report_path=cls.test_report, compression=UNCOMPRESSED, provider_id=cls.provider_id) cls.date_accessor = DateAccessor() billing_start = cls.date_accessor.today_with_timezone('UTC').replace( year=2018, month=6, day=1, hour=0, minute=0, second=0) cls.assembly_id = '1234' cls.manifest_dict = { 'assembly_id': cls.assembly_id, 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': 1 } cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map) cls.report_schema = cls.accessor.report_schema cls.session = cls.accessor._session _report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report, 'r') as f: reader = csv.DictReader(f) cls.row = next(reader)
def create_ocp_usage_line_item(self, report_period, report, resource_id=None, null_cpu_usage=False): """Create an OCP usage line item database object for test.""" table_name = OCP_REPORT_TABLE_MAP['line_item'] data = self.create_columns_for_table(table_name) for key in data: if 'byte' in key: data[key] = data[key] * Decimal(pow(2, 30)) if resource_id: data['resource_id'] = resource_id data['report_period_id'] = report_period.id data['report_id'] = report.id if null_cpu_usage: data['pod_usage_cpu_core_seconds'] = None with OCPReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def create_awscostentrylineitem_daily_summary( self, account_id, schema, cost_entry_bill, usage_date=None ): """Create reporting_awscostentrylineitem_daily_summary object for test.""" table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] data = self.create_columns_for_table(table_name) if usage_date is None: usage_date = self.fake.past_datetime() with AccountAliasAccessor(account_id, schema) as accessor: account_alias = accessor._get_db_obj_query().first() data = { 'account_alias_id': account_alias.id, 'cost_entry_bill': cost_entry_bill, 'usage_start': self.make_datetime_aware(usage_date), 'product_code': self.fake.pystr()[:8], 'usage_account_id': self.fake.pystr()[:8], } with OCPReportDBAccessor(self.schema, self.column_map) as accessor: obj = accessor.create_db_object(table_name, data) return obj
def test_update_summary_markup_charge(self): markup = {'value': 10, 'unit': 'percent'} markup_percentage = Decimal(markup.get('value')) / 100 rate = [{ 'metric': { 'name': 'memory_gb_usage_per_hour' }, 'tiered_rates': [{ 'value': 1, 'unit': 'USD' }] }] self.creator.create_cost_model(self.ocp_provider_uuid, 'OCP', rates=rate, markup=markup) usage_period = self.accessor.get_current_usage_period() start_date = usage_period.report_period_start.date() + relativedelta( days=-1) end_date = usage_period.report_period_end.date() + relativedelta( days=+1) self.accessor.populate_line_item_daily_table(start_date, end_date, self.cluster_id) self.accessor.populate_line_item_daily_summary_table( start_date, end_date, self.cluster_id) self.updater.update_summary_charge_info() table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary'] with OCPReportDBAccessor(schema=self.schema, column_map=self.column_map) as accessor: with schema_context(self.schema): items = accessor._get_db_obj_query(table_name).all() for item in items: markup_value = item.markup_cost self.assertEqual( markup_value, item.pod_charge_memory_gigabyte_hours * markup_percentage)
def _update_monthly_cost(self, start_date, end_date): """Update the monthly cost for a period of time.""" try: with OCPReportDBAccessor(self._schema) as report_accessor: # Ex. cost_type == "Node", rate_term == "node_cost_per_month", rate == 1000 for cost_type, rate_term in OCPUsageLineItemDailySummary.MONTHLY_COST_RATE_MAP.items( ): rate_type = None rate = None if self._infra_rates.get(rate_term): rate_type = metric_constants.INFRASTRUCTURE_COST_TYPE rate = self._infra_rates.get(rate_term) elif self._supplementary_rates.get(rate_term): rate_type = metric_constants.SUPPLEMENTARY_COST_TYPE rate = self._supplementary_rates.get(rate_term) log_msg = "Updating" if rate is None: log_msg = "Removing" LOG.info( log_msg + " monthly %s cost for" "\n\tSchema: %s \n\t%s Provider: %s (%s) \n\tDates: %s - %s", cost_type, self._schema, self._provider.type, self._provider.name, self._provider_uuid, start_date, end_date, ) report_accessor.populate_monthly_cost( cost_type, rate_type, rate, start_date, end_date, self._cluster_id, self._cluster_alias) except OCPCostModelCostUpdaterError as error: LOG.error("Unable to update monthly costs. Error: %s", str(error))
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() # These test reports should be replaced with OCP reports once processor is implemented. cls.test_report_path = ( "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv" ) cls.storage_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv" cls.node_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_node_labels.csv" cls.unknown_report = "./koku/masu/test/data/test_cur.csv" cls.test_report_gzip_path = "./koku/masu/test/data/test_cur.csv.gz" cls.date_accessor = DateAccessor() cls.billing_start = cls.date_accessor.today_with_timezone( "UTC").replace(year=2018, month=6, day=1, hour=0, minute=0, second=0) cls.assembly_id = "1234" cls.manifest_accessor = ReportManifestDBAccessor() cls.accessor = OCPReportDBAccessor(cls.schema) cls.report_schema = cls.accessor.report_schema _report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report_path, "r") as f: reader = csv.DictReader(f) cls.row = next(reader) with open(cls.storage_report_path, "r") as f: reader = csv.DictReader(f) cls.storage_row = next(reader)
def _generate_ocp_on_aws_data(self): """Generate OCP and AWS data.""" creator = ReportObjectCreator(self.schema, self.column_map) bill_ids = [] today = DateAccessor().today_with_timezone('UTC') last_month = today - relativedelta(months=1) resource_id = 'i-12345' for cost_entry_date in (today, last_month): bill = creator.create_cost_entry_bill( provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date ) bill_ids.append(str(bill.id)) cost_entry = creator.create_cost_entry(bill, cost_entry_date) product = creator.create_cost_entry_product('Compute Instance') pricing = creator.create_cost_entry_pricing() reservation = creator.create_cost_entry_reservation() creator.create_cost_entry_line_item( bill, cost_entry, product, pricing, reservation, resource_id=resource_id ) with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: aws_accessor.populate_line_item_daily_table(last_month.date(), today.date(), bill_ids) cluster_id = self.ocp_provider_resource_name provider_uuid = self.ocp_provider_uuid for cost_entry_date in (today, last_month): period = creator.create_ocp_report_period( provider_uuid=provider_uuid, period_date=cost_entry_date, cluster_id=cluster_id ) report = creator.create_ocp_report(period, cost_entry_date) creator.create_ocp_usage_line_item(period, report, resource_id=resource_id) cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid) with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: ocp_accessor.populate_line_item_daily_table(last_month.date(), today.date(), cluster_id)
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) aws_bills = get_bills_from_provider(self._provider.uuid, self._schema_name, start_date, end_date) aws_bill_ids = [str(bill.id) for bill in aws_bills] # OpenShift on AWS with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on AWS summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_ocp_on_aws_cost_daily_summary( start_date, end_date, cluster_id, aws_bill_ids) accessor.commit() if cluster_id: with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date) accessor.commit()
def _update_monthly_cost(self, start_date, end_date): """Update the monthly cost for a period of time.""" try: with CostModelDBAccessor(self._schema, self._provider_uuid, self._column_map) as cost_model_accessor, \ OCPReportDBAccessor(self._schema, self._column_map) as report_accessor: rates = cost_model_accessor.get_node_per_month_rates() if rates: tiers = self._normalize_tier(rates.get('tiered_rates', [])) for tier in tiers: LOG.info('Updating Monthly Cost for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema, self._provider_uuid, start_date, end_date) node_rate = Decimal(tier.get('value')) report_accessor.populate_monthly_cost(node_rate, start_date, end_date) else: LOG.info('Removing Monthly Cost for' 'Schema: %s, Provider: %s ', self._schema, self._provider_uuid) report_accessor.remove_monthly_cost() except OCPReportChargeUpdaterError as error: LOG.error('Unable to update monthly costs. Error: %s', str(error))
def create_cost_model(self, provider_uuid, source_type, rates=[], markup={}): """Create an OCP rate database object for test.""" table_name = OCP_REPORT_TABLE_MAP['cost_model'] cost_model_map = OCP_REPORT_TABLE_MAP['cost_model_map'] data = { 'uuid': str(uuid.uuid4()), 'created_timestamp': DateAccessor().today_with_timezone('UTC'), 'updated_timestamp': DateAccessor().today_with_timezone('UTC'), 'name': self.fake.pystr()[:8], 'description': self.fake.pystr(), 'source_type': source_type, 'rates': rates, 'markup': markup, } with ProviderDBAccessor(provider_uuid) as accessor: provider_obj = accessor.get_provider() with OCPReportDBAccessor(self.schema, self.column_map) as accessor: cost_model_obj = accessor.create_db_object(table_name, data) data = {'provider_uuid': provider_obj.uuid, 'cost_model_id': cost_model_obj.uuid} accessor.create_db_object(cost_model_map, data) return cost_model_obj
def create_cost_model(self, provider_uuid, source_type, rates=[], markup={}): """Create an OCP rate database object for test.""" table_name = OCP_REPORT_TABLE_MAP["cost_model"] cost_model_map = OCP_REPORT_TABLE_MAP["cost_model_map"] data = { "uuid": str(uuid.uuid4()), "created_timestamp": DateAccessor().today_with_timezone("UTC"), "updated_timestamp": DateAccessor().today_with_timezone("UTC"), "name": self.fake.pystr()[:8], "description": self.fake.pystr(), "source_type": source_type, "rates": rates, "markup": markup, } with ProviderDBAccessor(provider_uuid) as accessor: provider_obj = accessor.get_provider() with OCPReportDBAccessor(self.schema) as accessor: cost_model_obj = accessor.create_db_object(table_name, data) data = {"provider_uuid": provider_obj.uuid, "cost_model_id": cost_model_obj.uuid} accessor.create_db_object(cost_model_map, data) return cost_model_obj
def __init__(self, schema_name, report_path, compression, provider_id): """Initialize base class.""" super().__init__(schema_name=schema_name, report_path=report_path, compression=compression, provider_id=provider_id) self._report_name = path.basename(report_path) self._cluster_id = report_path.split('/')[-2] self._datetime_format = Config.OCP_DATETIME_STR_FORMAT self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE with ReportingCommonDBAccessor() as report_common_db: self.column_map = report_common_db.column_map with OCPReportDBAccessor(self._schema_name, self.column_map) as report_db: self.existing_report_periods_map = report_db.get_report_periods() self.existing_report_map = report_db.get_reports() self.line_item_columns = None self.processed_report = ProcessedOCPReport()
def _update_markup_cost(self, start_date, end_date): """Populate markup costs for OpenShift. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ infra_markup_value = Decimal(0.0) infra_map = self.get_infra_map() infra_tuple = infra_map.get(self._provider_uuid) cluster_id = get_cluster_id_from_provider(self._provider_uuid) if infra_tuple: infra_uuid = infra_tuple[0] with CostModelDBAccessor(self._schema, infra_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() infra_markup_value = Decimal(markup.get("value", 0)) / 100 with CostModelDBAccessor(self._schema, self._provider_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() ocp_markup_value = Decimal(markup.get("value", 0)) / 100 with OCPReportDBAccessor(self._schema, self._column_map) as accessor: LOG.info( "Updating OpenShift markup for" "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s", self._schema, self._provider_uuid, start_date, end_date, ) accessor.populate_markup_cost(infra_markup_value, ocp_markup_value, cluster_id) LOG.info("Finished updating markup.")
def test_update_summary_tables_azure(self, mock_cost_model): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} mock_cost_model.markup = markup start_date = self.dh.this_month_start end_date = self.dh.this_month_end updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.azure_provider, manifest=None) updater.update_summary_tables(start_date, end_date) summary_table_name = AZURE_REPORT_TABLE_MAP["ocp_on_azure_daily_summary"] with AzureReportDBAccessor(self.schema) as azure_accessor: query = azure_accessor._get_db_obj_query(summary_table_name).filter( cost_entry_bill__billing_period_start=start_date ) markup_cost = query.aggregate(Sum("markup_cost"))["markup_cost__sum"] pretax_cost = query.aggregate(Sum("pretax_cost"))["pretax_cost__sum"] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get("value") / 100), places=5) daily_summary_table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"] with OCPReportDBAccessor(self.schema) as ocp_accessor: query = ocp_accessor._get_db_obj_query(daily_summary_table_name).filter( report_period__provider=self.ocp_on_azure_ocp_provider, report_period__report_period_start=self.dh.this_month_start, ) infra_cost = query.aggregate(Sum("infrastructure_raw_cost"))["infrastructure_raw_cost__sum"] project_infra_cost = query.aggregate(Sum("infrastructure_project_raw_cost"))[ "infrastructure_project_raw_cost__sum" ] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def _update_monthly_cost(self, start_date, end_date): """Update the monthly cost for a period of time.""" try: with CostModelDBAccessor( self._schema, self._provider_uuid, self._column_map ) as cost_model_accessor, OCPReportDBAccessor( self._schema, self._column_map) as report_accessor: rates = cost_model_accessor.get_node_per_month_rates() if rates: # tiers = self._normalize_tier(rates.get('tiered_rates', [])) # FIXME: The _normalize_tier function is currently return two # rates, one for the first of the current month & one for the # first of the next month. # example: [{'unit': 'USD', 'value': '0.2'}, {'unit': 'USD', 'value': '0.2'}] tiers = rates.get("tiered_rates", []) for tier in tiers: LOG.info( "Updating Monthly Cost for" "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s", self._schema, self._provider_uuid, start_date, end_date, ) node_rate = Decimal(tier.get("value")) report_accessor.populate_monthly_cost( node_rate, start_date, end_date, self._cluster_id, self._cluster_alias) else: LOG.info( "Removing Monthly Cost for" "Schema: %s, Provider: %s ", self._schema, self._provider_uuid) report_accessor.remove_monthly_cost() except OCPReportChargeUpdaterError as error: LOG.error("Unable to update monthly costs. Error: %s", str(error))
def update_cost_summary_table(self, start_date, end_date): """Populate the cost summary tables. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) # This needs to always run regardless of whether the OpenShift # cluster is tied to a cloud provider infra_map = self._get_ocp_infra_map(start_date, end_date) aws_uuid = self._get_aws_provider_uuid_from_map( self._provider, infra_map) with CostModelDBAccessor(self._schema_name, aws_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() aws_markup_value = float(markup.get('value', 0)) / 100 with CostModelDBAccessor(self._schema_name, self._provider.uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() ocp_markup_value = float(markup.get('value', 0)) / 100 with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date) accessor.populate_ocp_on_aws_markup_cost(aws_markup_value, ocp_markup_value, cluster_id)
def update_summary_charge_info(self, start_date, end_date): """Update the OCP summary table with the charge information. Args: start_date (str, Optional) - Start date of range to update derived cost. end_date (str, Optional) - End date of range to update derived cost. Returns None """ if isinstance(start_date, str): start_date = parse(start_date) if isinstance(end_date, str): end_date = parse(end_date) self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) self._cluster_alias = get_cluster_alias_from_cluster_id( self._cluster_id) LOG.info( "Starting charge calculation updates for provider: %s. Cluster ID: %s.", self._provider_uuid, self._cluster_id, ) self._update_pod_charge(start_date, end_date) self._update_storage_charge(start_date, end_date) self._update_markup_cost(start_date, end_date) self._update_monthly_cost(start_date, end_date) with OCPReportDBAccessor(self._schema, self._column_map) as accessor: report_periods = accessor.report_periods_for_provider_uuid( self._provider_uuid, start_date) with schema_context(self._schema): for period in report_periods: period.derived_cost_datetime = DateAccessor( ).today_with_timezone("UTC") period.save()
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() # These test reports should be replaced with OCP reports once processor is impelmented. cls.test_report = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv' cls.storage_report = ( './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv' ) cls.unknown_report = './koku/masu/test/data/test_cur.csv' cls.test_report_gzip = './koku/masu/test/data/test_cur.csv.gz' cls.date_accessor = DateAccessor() cls.billing_start = cls.date_accessor.today_with_timezone( 'UTC').replace(year=2018, month=6, day=1, hour=0, minute=0, second=0) cls.assembly_id = '1234' cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = OCPReportDBAccessor(cls.schema, cls.column_map) cls.report_schema = cls.accessor.report_schema _report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report, 'r') as f: reader = csv.DictReader(f) cls.row = next(reader)
def create_ocpawscostlineitem_project_daily_summary( self, account_id, schema): """Create an ocpawscostlineitem_project_daily_summary object for test.""" table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_project_daily_summary'] data = self.create_columns_for_table(table_name) with AccountAliasAccessor(account_id, schema) as accessor: account_alias = accessor._get_db_obj_query().first() data = { 'account_alias_id': account_alias.id, 'cost_entry_bill': self.create_cost_entry_bill(), 'namespace': self.fake.pystr()[:8], 'pod': self.fake.pystr()[:8], 'node': self.fake.pystr()[:8], 'usage_start': self.make_datetime_aware(self.fake.past_datetime()), 'usage_end': self.make_datetime_aware(self.fake.past_datetime()), 'product_code': self.fake.pystr()[:8], 'usage_account_id': self.fake.pystr()[:8] } with OCPReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)