def create_ocpawscostlineitem_project_daily_summary(self, account_id, schema):
"""Create an ocpawscostlineitem_project_daily_summary object for test."""
table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_project_daily_summary"]
data = self.create_columns_for_table(table_name)
with AccountAliasAccessor(account_id, schema) as accessor:
account_alias = accessor._get_db_obj_query().first()
data = {
"account_alias_id": account_alias.id,
"cost_entry_bill": self.create_cost_entry_bill(str(uuid.uuid4())),
"namespace": self.fake.pystr()[:8],
"pod": self.fake.pystr()[:8],
"node": self.fake.pystr()[:8],
"usage_start": self.make_datetime_aware(self.fake.past_datetime()),
"usage_end": self.make_datetime_aware(self.fake.past_datetime()),
"product_code": self.fake.pystr()[:8],
"usage_account_id": self.fake.pystr()[:8],
}
with OCPReportDBAccessor(self.schema) as accessor:
return accessor.create_db_object(table_name, data)
def create_ocp_report_period(self, provider_uuid, period_date=None, cluster_id=None):
"""Create an OCP report database object for test."""
table_name = OCP_REPORT_TABLE_MAP['report_period']
period_start = self.make_datetime_aware(self.fake.past_datetime()).date().replace(day=1)
period_end = period_start + relativedelta.relativedelta(days=random.randint(1, 15))
data = {
'cluster_id': cluster_id if cluster_id else self.fake.pystr()[:8],
'provider_id': provider_uuid,
'report_period_start': period_start,
'report_period_end': period_end,
}
if period_date:
period_start = period_date.replace(day=1).date()
period_end = period_start + relativedelta.relativedelta(months=1)
data['report_period_start'] = period_start
data['report_period_end'] = period_end
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def create_ocp_storage_line_item(self,
report_period,
report,
pod=None,
namespace=None):
"""Create an OCP storage line item database object for test."""
table_name = OCP_REPORT_TABLE_MAP['storage_line_item']
data = self.create_columns_for_table(table_name)
for key in data:
if 'bytes' in key or 'byte' in key:
data[key] = data[key] * Decimal(pow(2, 30)) * 5
data['report_period_id'] = report_period.id
data['report_id'] = report.id
if pod:
data['pod'] = pod
if namespace:
data['namespace'] = namespace
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
LOG.info(
'Updating OpenShift report summary tables for \n\tSchema: %s '
'\n\tProvider: %s \n\tCluster: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, self._cluster_id,
start_date, end_date)
report_periods = None
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
report_periods = accessor.report_periods_for_provider_id(
self._provider.id, start_date)
accessor.populate_line_item_daily_summary_table(
start_date, end_date, self._cluster_id)
accessor.populate_pod_label_summary_table()
accessor.populate_storage_line_item_daily_summary_table(
start_date, end_date, self._cluster_id)
accessor.populate_volume_claim_label_summary_table()
accessor.populate_volume_label_summary_table()
for period in report_periods:
if period.summary_data_creation_datetime is None:
period.summary_data_creation_datetime = \
self._date_accessor.today_with_timezone('UTC')
period.summary_data_updated_datetime = \
self._date_accessor.today_with_timezone('UTC')
period.save()
return start_date, end_date
def test_update_aws_summary_tables(self, mock_utility, mock_ocp_on_aws,
mock_tag_summary, mock_map):
"""Test that summary tables are properly run for an OCP provider."""
fake_bills = MagicMock()
fake_bills.__iter__.return_value = [Mock(), Mock()]
first = Mock()
bill_id = 1
first.return_value.id = bill_id
fake_bills.first = first
mock_utility.return_value = fake_bills
start_date = self.dh.today.date()
end_date = start_date + datetime.timedelta(days=1)
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with OCPReportDBAccessor(self.schema_name) as accessor:
report_period = accessor.report_periods_for_provider_uuid(
self.ocp_test_provider_uuid, start_date)
with schema_context(self.schema_name):
current_ocp_report_period_id = report_period.id
mock_map.return_value = {
self.ocp_test_provider_uuid:
(self.aws_provider_uuid, Provider.PROVIDER_AWS)
}
updater = OCPCloudParquetReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
updater.update_aws_summary_tables(self.ocp_test_provider_uuid,
self.aws_test_provider_uuid,
start_date, end_date)
mock_ocp_on_aws.assert_called_with(
start_date,
end_date,
self.ocp_test_provider_uuid,
self.aws_test_provider_uuid,
current_ocp_report_period_id,
bill_id,
decimal.Decimal(0),
)
def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
if isinstance(start_date, str):
start_date = parser.parse(start_date)
if isinstance(end_date, str):
end_date = parser.parse(end_date)
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
aws_bills = aws_get_bills_from_provider(aws_provider_uuid, self._schema, start_date, end_date)
aws_bill_ids = []
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
with CostModelDBAccessor(self._schema, aws_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating OpenShift on AWS summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, AWS Bill IDs: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
str(aws_bill_ids),
)
accessor.populate_ocp_on_aws_cost_daily_summary(start, end, cluster_id, aws_bill_ids)
accessor.populate_ocp_on_aws_markup_cost(markup_value, aws_bill_ids)
accessor.populate_ocp_on_aws_tags_summary_table()
self.refresh_openshift_on_infrastructure_views(OCP_ON_AWS_MATERIALIZED_VIEWS)
with OCPReportDBAccessor(self._schema) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(cluster_id, start_date, end_date)
def test_update_summary_tables_existing_period_done_processing(
self, mock_daily, mock_sum, mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are not run for a full month."""
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_creation_datetime = start_date
self.accessor.commit()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor('acct10001', self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
def __init__(self, schema_name, report_path, compression, provider_uuid):
"""Initialize base class."""
super().__init__(
schema_name=schema_name,
report_path=report_path,
compression=compression,
provider_uuid=provider_uuid,
manifest_id=None,
processed_report=ProcessedOCPReport(),
)
self._report_name = path.basename(report_path)
self._cluster_id = utils.get_cluster_id_from_provider(provider_uuid)
self._datetime_format = Config.OCP_DATETIME_STR_FORMAT
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
with OCPReportDBAccessor(self._schema) as report_db:
self.existing_report_periods_map = report_db.get_report_periods()
self.existing_report_map = report_db.get_reports()
self.line_item_columns = None
def update_cost_summary_table(self, start_date, end_date):
"""Populate the cost summary tables.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
# This needs to always run regardless of whether the OpenShift
# cluster is tied to a cloud provider
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
def test_update_summary_tables_without_manifest(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are properly run without a manifest."""
# Create an updater that doesn't have a manifest
updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
start_date = DateAccessor().today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_updated_datetime = start_date
period.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(start_date.date(), end_date.date(),
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(start_date.date(),
end_date.date(),
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(start_date.date(), end_date.date(),
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(start_date.date(),
end_date.date(),
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertGreater(period.summary_data_updated_datetime,
self.today)
def process(self, parquet_base_filename, daily_data_frames):
"""Filter data and convert to parquet."""
for ocp_provider_uuid, infra_tuple in self.ocp_infrastructure_map.items():
infra_provider_uuid = infra_tuple[0]
if infra_provider_uuid != self.provider_uuid:
continue
msg = (
f"Processing OpenShift on {self.provider_type} to parquet."
f"\n\tStart date: {str(self.start_date)}\n\tFile: {str(self.report_file)}"
)
LOG.info(msg)
# Get OpenShift topology data
with OCPReportDBAccessor(self.schema_name) as accessor:
if not accessor.get_cluster_for_provider(ocp_provider_uuid):
LOG.info(
f"No cluster information available for OCP Provider: {ocp_provider_uuid},"
+ "skipping OCP on Cloud parquet processing."
)
continue
cluster_topology = accessor.get_openshift_topology_for_provider(ocp_provider_uuid)
# Get matching tags
report_period_id = self.get_report_period_id(ocp_provider_uuid)
matched_tags = []
if self.has_enabled_ocp_labels:
LOG.info("Getting matching tags from Postgres.")
matched_tags = self.db_accessor.get_openshift_on_cloud_matched_tags(self.bill_id, report_period_id)
if not matched_tags:
LOG.info("Matched tags not yet available via Postgres. Getting matching tags from Trino.")
matched_tags = self.db_accessor.get_openshift_on_cloud_matched_tags_trino(
self.provider_uuid, ocp_provider_uuid, self.start_date, self.end_date
)
for i, daily_data_frame in enumerate(daily_data_frames):
openshift_filtered_data_frame = self.ocp_on_cloud_data_processor(
daily_data_frame, cluster_topology, matched_tags
)
self.create_ocp_on_cloud_parquet(
openshift_filtered_data_frame, parquet_base_filename, i, ocp_provider_uuid
)
def setUpClass(cls):
"""Set up the test class with required objects."""
# These test reports should be replaced with OCP reports once processor is impelmented.
cls.test_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
cls.storage_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv'
cls.unknown_report = './tests/data/test_cur.csv'
cls.test_report_gzip = './tests/data/test_cur.csv.gz'
cls.provider_id = 1
cls.ocp_processor = OCPReportProcessor(schema_name='acct10001',
report_path=cls.test_report,
compression=UNCOMPRESSED,
provider_id=cls.provider_id)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
cls.assembly_id = '1234'
cls.manifest_dict = {
'assembly_id': cls.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': 1
}
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def create_ocp_usage_line_item(self,
report_period,
report,
resource_id=None,
null_cpu_usage=False):
"""Create an OCP usage line item database object for test."""
table_name = OCP_REPORT_TABLE_MAP['line_item']
data = self.create_columns_for_table(table_name)
for key in data:
if 'byte' in key:
data[key] = data[key] * Decimal(pow(2, 30))
if resource_id:
data['resource_id'] = resource_id
data['report_period_id'] = report_period.id
data['report_id'] = report.id
if null_cpu_usage:
data['pod_usage_cpu_core_seconds'] = None
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def create_awscostentrylineitem_daily_summary(
self, account_id, schema, cost_entry_bill, usage_date=None
):
"""Create reporting_awscostentrylineitem_daily_summary object for test."""
table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary']
data = self.create_columns_for_table(table_name)
if usage_date is None:
usage_date = self.fake.past_datetime()
with AccountAliasAccessor(account_id, schema) as accessor:
account_alias = accessor._get_db_obj_query().first()
data = {
'account_alias_id': account_alias.id,
'cost_entry_bill': cost_entry_bill,
'usage_start': self.make_datetime_aware(usage_date),
'product_code': self.fake.pystr()[:8],
'usage_account_id': self.fake.pystr()[:8],
}
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
obj = accessor.create_db_object(table_name, data)
return obj
def test_update_summary_markup_charge(self):
markup = {'value': 10, 'unit': 'percent'}
markup_percentage = Decimal(markup.get('value')) / 100
rate = [{
'metric': {
'name': 'memory_gb_usage_per_hour'
},
'tiered_rates': [{
'value': 1,
'unit': 'USD'
}]
}]
self.creator.create_cost_model(self.ocp_provider_uuid,
'OCP',
rates=rate,
markup=markup)
usage_period = self.accessor.get_current_usage_period()
start_date = usage_period.report_period_start.date() + relativedelta(
days=-1)
end_date = usage_period.report_period_end.date() + relativedelta(
days=+1)
self.accessor.populate_line_item_daily_table(start_date, end_date,
self.cluster_id)
self.accessor.populate_line_item_daily_summary_table(
start_date, end_date, self.cluster_id)
self.updater.update_summary_charge_info()
table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']
with OCPReportDBAccessor(schema=self.schema,
column_map=self.column_map) as accessor:
with schema_context(self.schema):
items = accessor._get_db_obj_query(table_name).all()
for item in items:
markup_value = item.markup_cost
self.assertEqual(
markup_value, item.pod_charge_memory_gigabyte_hours *
markup_percentage)
def _update_monthly_cost(self, start_date, end_date):
"""Update the monthly cost for a period of time."""
try:
with OCPReportDBAccessor(self._schema) as report_accessor:
# Ex. cost_type == "Node", rate_term == "node_cost_per_month", rate == 1000
for cost_type, rate_term in OCPUsageLineItemDailySummary.MONTHLY_COST_RATE_MAP.items(
):
rate_type = None
rate = None
if self._infra_rates.get(rate_term):
rate_type = metric_constants.INFRASTRUCTURE_COST_TYPE
rate = self._infra_rates.get(rate_term)
elif self._supplementary_rates.get(rate_term):
rate_type = metric_constants.SUPPLEMENTARY_COST_TYPE
rate = self._supplementary_rates.get(rate_term)
log_msg = "Updating"
if rate is None:
log_msg = "Removing"
LOG.info(
log_msg + " monthly %s cost for"
"\n\tSchema: %s \n\t%s Provider: %s (%s) \n\tDates: %s - %s",
cost_type,
self._schema,
self._provider.type,
self._provider.name,
self._provider_uuid,
start_date,
end_date,
)
report_accessor.populate_monthly_cost(
cost_type, rate_type, rate, start_date, end_date,
self._cluster_id, self._cluster_alias)
except OCPCostModelCostUpdaterError as error:
LOG.error("Unable to update monthly costs. Error: %s", str(error))
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
# These test reports should be replaced with OCP reports once processor is implemented.
cls.test_report_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv"
)
cls.storage_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv"
cls.node_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_node_labels.csv"
cls.unknown_report = "./koku/masu/test/data/test_cur.csv"
cls.test_report_gzip_path = "./koku/masu/test/data/test_cur.csv.gz"
cls.date_accessor = DateAccessor()
cls.billing_start = cls.date_accessor.today_with_timezone(
"UTC").replace(year=2018,
month=6,
day=1,
hour=0,
minute=0,
second=0)
cls.assembly_id = "1234"
cls.manifest_accessor = ReportManifestDBAccessor()
cls.accessor = OCPReportDBAccessor(cls.schema)
cls.report_schema = cls.accessor.report_schema
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_path, "r") as f:
reader = csv.DictReader(f)
cls.row = next(reader)
with open(cls.storage_report_path, "r") as f:
reader = csv.DictReader(f)
cls.storage_row = next(reader)
def _generate_ocp_on_aws_data(self):
"""Generate OCP and AWS data."""
creator = ReportObjectCreator(self.schema, self.column_map)
bill_ids = []
today = DateAccessor().today_with_timezone('UTC')
last_month = today - relativedelta(months=1)
resource_id = 'i-12345'
for cost_entry_date in (today, last_month):
bill = creator.create_cost_entry_bill(
provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date
)
bill_ids.append(str(bill.id))
cost_entry = creator.create_cost_entry(bill, cost_entry_date)
product = creator.create_cost_entry_product('Compute Instance')
pricing = creator.create_cost_entry_pricing()
reservation = creator.create_cost_entry_reservation()
creator.create_cost_entry_line_item(
bill, cost_entry, product, pricing, reservation, resource_id=resource_id
)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
aws_accessor.populate_line_item_daily_table(last_month.date(), today.date(), bill_ids)
cluster_id = self.ocp_provider_resource_name
provider_uuid = self.ocp_provider_uuid
for cost_entry_date in (today, last_month):
period = creator.create_ocp_report_period(
provider_uuid=provider_uuid, period_date=cost_entry_date, cluster_id=cluster_id
)
report = creator.create_ocp_report(period, cost_entry_date)
creator.create_ocp_usage_line_item(period, report, resource_id=resource_id)
cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
ocp_accessor.populate_line_item_daily_table(last_month.date(), today.date(), cluster_id)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
aws_bills = get_bills_from_provider(self._provider.uuid,
self._schema_name, start_date,
end_date)
aws_bill_ids = [str(bill.id) for bill in aws_bills]
# OpenShift on AWS
with AWSReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.commit()
if cluster_id:
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date,
end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
accessor.commit()
def _update_monthly_cost(self, start_date, end_date):
"""Update the monthly cost for a period of time."""
try:
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor, \
OCPReportDBAccessor(self._schema, self._column_map) as report_accessor:
rates = cost_model_accessor.get_node_per_month_rates()
if rates:
tiers = self._normalize_tier(rates.get('tiered_rates', []))
for tier in tiers:
LOG.info('Updating Monthly Cost for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema, self._provider_uuid, start_date, end_date)
node_rate = Decimal(tier.get('value'))
report_accessor.populate_monthly_cost(node_rate, start_date, end_date)
else:
LOG.info('Removing Monthly Cost for'
'Schema: %s, Provider: %s ',
self._schema, self._provider_uuid)
report_accessor.remove_monthly_cost()
except OCPReportChargeUpdaterError as error:
LOG.error('Unable to update monthly costs. Error: %s', str(error))
def create_cost_model(self, provider_uuid, source_type, rates=[], markup={}):
"""Create an OCP rate database object for test."""
table_name = OCP_REPORT_TABLE_MAP['cost_model']
cost_model_map = OCP_REPORT_TABLE_MAP['cost_model_map']
data = {
'uuid': str(uuid.uuid4()),
'created_timestamp': DateAccessor().today_with_timezone('UTC'),
'updated_timestamp': DateAccessor().today_with_timezone('UTC'),
'name': self.fake.pystr()[:8],
'description': self.fake.pystr(),
'source_type': source_type,
'rates': rates,
'markup': markup,
}
with ProviderDBAccessor(provider_uuid) as accessor:
provider_obj = accessor.get_provider()
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
cost_model_obj = accessor.create_db_object(table_name, data)
data = {'provider_uuid': provider_obj.uuid, 'cost_model_id': cost_model_obj.uuid}
accessor.create_db_object(cost_model_map, data)
return cost_model_obj
def create_cost_model(self, provider_uuid, source_type, rates=[], markup={}):
"""Create an OCP rate database object for test."""
table_name = OCP_REPORT_TABLE_MAP["cost_model"]
cost_model_map = OCP_REPORT_TABLE_MAP["cost_model_map"]
data = {
"uuid": str(uuid.uuid4()),
"created_timestamp": DateAccessor().today_with_timezone("UTC"),
"updated_timestamp": DateAccessor().today_with_timezone("UTC"),
"name": self.fake.pystr()[:8],
"description": self.fake.pystr(),
"source_type": source_type,
"rates": rates,
"markup": markup,
}
with ProviderDBAccessor(provider_uuid) as accessor:
provider_obj = accessor.get_provider()
with OCPReportDBAccessor(self.schema) as accessor:
cost_model_obj = accessor.create_db_object(table_name, data)
data = {"provider_uuid": provider_obj.uuid, "cost_model_id": cost_model_obj.uuid}
accessor.create_db_object(cost_model_map, data)
return cost_model_obj
def __init__(self, schema_name, report_path, compression, provider_id):
"""Initialize base class."""
super().__init__(schema_name=schema_name,
report_path=report_path,
compression=compression,
provider_id=provider_id)
self._report_name = path.basename(report_path)
self._cluster_id = report_path.split('/')[-2]
self._datetime_format = Config.OCP_DATETIME_STR_FORMAT
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
with ReportingCommonDBAccessor() as report_common_db:
self.column_map = report_common_db.column_map
with OCPReportDBAccessor(self._schema_name,
self.column_map) as report_db:
self.existing_report_periods_map = report_db.get_report_periods()
self.existing_report_map = report_db.get_reports()
self.line_item_columns = None
self.processed_report = ProcessedOCPReport()
def _update_markup_cost(self, start_date, end_date):
"""Populate markup costs for OpenShift.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
infra_markup_value = Decimal(0.0)
infra_map = self.get_infra_map()
infra_tuple = infra_map.get(self._provider_uuid)
cluster_id = get_cluster_id_from_provider(self._provider_uuid)
if infra_tuple:
infra_uuid = infra_tuple[0]
with CostModelDBAccessor(self._schema, infra_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
infra_markup_value = Decimal(markup.get("value", 0)) / 100
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
ocp_markup_value = Decimal(markup.get("value", 0)) / 100
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
"Updating OpenShift markup for"
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider_uuid,
start_date,
end_date,
)
accessor.populate_markup_cost(infra_markup_value, ocp_markup_value,
cluster_id)
LOG.info("Finished updating markup.")
def test_update_summary_tables_azure(self, mock_cost_model):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_cost_model.markup = markup
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.azure_provider, manifest=None)
updater.update_summary_tables(start_date, end_date)
summary_table_name = AZURE_REPORT_TABLE_MAP["ocp_on_azure_daily_summary"]
with AzureReportDBAccessor(self.schema) as azure_accessor:
query = azure_accessor._get_db_obj_query(summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date
)
markup_cost = query.aggregate(Sum("markup_cost"))["markup_cost__sum"]
pretax_cost = query.aggregate(Sum("pretax_cost"))["pretax_cost__sum"]
self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get("value") / 100), places=5)
daily_summary_table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]
with OCPReportDBAccessor(self.schema) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(daily_summary_table_name).filter(
report_period__provider=self.ocp_on_azure_ocp_provider,
report_period__report_period_start=self.dh.this_month_start,
)
infra_cost = query.aggregate(Sum("infrastructure_raw_cost"))["infrastructure_raw_cost__sum"]
project_infra_cost = query.aggregate(Sum("infrastructure_project_raw_cost"))[
"infrastructure_project_raw_cost__sum"
]
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def _update_monthly_cost(self, start_date, end_date):
"""Update the monthly cost for a period of time."""
try:
with CostModelDBAccessor(
self._schema, self._provider_uuid, self._column_map
) as cost_model_accessor, OCPReportDBAccessor(
self._schema, self._column_map) as report_accessor:
rates = cost_model_accessor.get_node_per_month_rates()
if rates:
# tiers = self._normalize_tier(rates.get('tiered_rates', []))
# FIXME: The _normalize_tier function is currently return two
# rates, one for the first of the current month & one for the
# first of the next month.
# example: [{'unit': 'USD', 'value': '0.2'}, {'unit': 'USD', 'value': '0.2'}]
tiers = rates.get("tiered_rates", [])
for tier in tiers:
LOG.info(
"Updating Monthly Cost for"
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider_uuid,
start_date,
end_date,
)
node_rate = Decimal(tier.get("value"))
report_accessor.populate_monthly_cost(
node_rate, start_date, end_date, self._cluster_id,
self._cluster_alias)
else:
LOG.info(
"Removing Monthly Cost for"
"Schema: %s, Provider: %s ", self._schema,
self._provider_uuid)
report_accessor.remove_monthly_cost()
except OCPReportChargeUpdaterError as error:
LOG.error("Unable to update monthly costs. Error: %s", str(error))
def update_cost_summary_table(self, start_date, end_date):
"""Populate the cost summary tables.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
# This needs to always run regardless of whether the OpenShift
# cluster is tied to a cloud provider
infra_map = self._get_ocp_infra_map(start_date, end_date)
aws_uuid = self._get_aws_provider_uuid_from_map(
self._provider, infra_map)
with CostModelDBAccessor(self._schema_name, aws_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
aws_markup_value = float(markup.get('value', 0)) / 100
with CostModelDBAccessor(self._schema_name, self._provider.uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
ocp_markup_value = float(markup.get('value', 0)) / 100
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
accessor.populate_ocp_on_aws_markup_cost(aws_markup_value,
ocp_markup_value,
cluster_id)
def update_summary_charge_info(self, start_date, end_date):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
if isinstance(start_date, str):
start_date = parse(start_date)
if isinstance(end_date, str):
end_date = parse(end_date)
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
self._cluster_alias = get_cluster_alias_from_cluster_id(
self._cluster_id)
LOG.info(
"Starting charge calculation updates for provider: %s. Cluster ID: %s.",
self._provider_uuid,
self._cluster_id,
)
self._update_pod_charge(start_date, end_date)
self._update_storage_charge(start_date, end_date)
self._update_markup_cost(start_date, end_date)
self._update_monthly_cost(start_date, end_date)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
report_periods = accessor.report_periods_for_provider_uuid(
self._provider_uuid, start_date)
with schema_context(self._schema):
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone("UTC")
period.save()
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
# These test reports should be replaced with OCP reports once processor is impelmented.
cls.test_report = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
cls.storage_report = (
'./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv'
)
cls.unknown_report = './koku/masu/test/data/test_cur.csv'
cls.test_report_gzip = './koku/masu/test/data/test_cur.csv.gz'
cls.date_accessor = DateAccessor()
cls.billing_start = cls.date_accessor.today_with_timezone(
'UTC').replace(year=2018,
month=6,
day=1,
hour=0,
minute=0,
second=0)
cls.assembly_id = '1234'
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor(cls.schema, cls.column_map)
cls.report_schema = cls.accessor.report_schema
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def create_ocpawscostlineitem_project_daily_summary(
self, account_id, schema):
"""Create an ocpawscostlineitem_project_daily_summary object for test."""
table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_project_daily_summary']
data = self.create_columns_for_table(table_name)
with AccountAliasAccessor(account_id, schema) as accessor:
account_alias = accessor._get_db_obj_query().first()
data = {
'account_alias_id': account_alias.id,
'cost_entry_bill': self.create_cost_entry_bill(),
'namespace': self.fake.pystr()[:8],
'pod': self.fake.pystr()[:8],
'node': self.fake.pystr()[:8],
'usage_start':
self.make_datetime_aware(self.fake.past_datetime()),
'usage_end':
self.make_datetime_aware(self.fake.past_datetime()),
'product_code': self.fake.pystr()[:8],
'usage_account_id': self.fake.pystr()[:8]
}
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
