def update_daily_tables(self, start_date, end_date):
"""Populate the daily tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
LOG.info(
'Updating OpenShift report daily tables for \n\tSchema: %s '
'\n\tProvider: %s \n\tCluster: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, self._cluster_id,
start_date, end_date)
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
accessor.populate_line_item_daily_table(start_date, end_date,
self._cluster_id)
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
accessor.populate_storage_line_item_daily_table(
start_date, end_date, self._cluster_id)
return start_date, end_date
def _generate_ocp_infra_map_from_sql(self, start_date, end_date):
"""Get the OCP on X infrastructure map.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns:
infra_map (dict) The OCP infrastructure map.
"""
infra_map = {}
if self._provider.type == Provider.PROVIDER_OCP:
with OCPReportDBAccessor(self._schema) as accessor:
infra_map = accessor.get_ocp_infrastructure_map(
start_date, end_date, ocp_provider_uuid=self._provider_uuid
)
elif self._provider.type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL):
with OCPReportDBAccessor(self._schema) as accessor:
infra_map = accessor.get_ocp_infrastructure_map(
start_date, end_date, aws_provider_uuid=self._provider_uuid
)
elif self._provider.type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL):
with OCPReportDBAccessor(self._schema) as accessor:
infra_map = accessor.get_ocp_infrastructure_map(
start_date, end_date, azure_provider_uuid=self._provider_uuid
)
# Save to DB
self.set_provider_infra_map(infra_map)
return infra_map
def test_update_cost_summary_table_for_markup(self, mock_markup):
"""Test that summary tables are updated correctly."""
markup = {'value': 10, 'unit': 'percent'}
mock_markup.return_value = markup
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
ocp_accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
cost_summary_table = OCP_REPORT_TABLE_MAP['cost_summary']
cost_summary_query = ocp_accessor._get_db_obj_query(
cost_summary_table)
possible_values = {}
with schema_context(self.schema):
for item in cost_summary_query:
possible_values.update({
item.cluster_id:
((item.pod_charge_cpu_core_hours +
item.pod_charge_memory_gigabyte_hours +
item.persistentvolumeclaim_charge_gb_month +
item.infra_cost) * decimal.Decimal(0.1))
})
updater.update_cost_summary_table(start_date_str, end_date_str)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(cost_summary_table)
found_values = {}
for item in query:
found_values.update({item.cluster_id: item.markup_cost})
for k, v in found_values.items():
self.assertAlmostEqual(v, possible_values[k], places=6)
def update_summary_charge_info(self, start_date=None, end_date=None):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
LOG.info(
'Starting charge calculation updates for provider: %s. Cluster ID: %s.',
self._provider_uuid, self._cluster_id)
self._update_pod_charge()
self._update_storage_charge()
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on Cloud cost summary for schema: %s and provider: %s',
self._schema, self._provider_uuid)
accessor.populate_cost_summary_table(self._cluster_id,
start_date=start_date,
end_date=end_date)
report_periods = accessor.report_periods_for_provider_id(
self._provider_id, start_date)
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone('UTC')
accessor.commit()
def test_update_summary_tables_with_manifest(self, mock_sum,
mock_storage_summary):
"""Test that summary tables are properly run."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_creation_datetime = start_date
period.summary_data_updated_datetime = None
period.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
self.assertIsNone(period.summary_data_updated_datetime)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called()
mock_storage_summary.assert_called()
with OCPReportDBAccessor(self.schema) as accessor:
period = accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
# These test reports should be replaced with OCP reports once processor is implemented.
cls.test_report_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv"
)
cls.storage_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv"
cls.node_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_node_labels.csv"
cls.unknown_report = "./koku/masu/test/data/test_cur.csv"
cls.test_report_gzip_path = "./koku/masu/test/data/test_cur.csv.gz"
cls.date_accessor = DateAccessor()
cls.billing_start = cls.date_accessor.today_with_timezone(
"UTC").replace(year=2018,
month=6,
day=1,
hour=0,
minute=0,
second=0)
cls.assembly_id = "1234"
cls.manifest_accessor = ReportManifestDBAccessor()
cls.accessor = OCPReportDBAccessor(cls.schema)
cls.report_schema = cls.accessor.report_schema
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_path, "r") as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.accessor = OCPReportDBAccessor(schema=cls.schema)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(cls.schema)
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
def create_cost_model(self,
provider_uuid,
source_type,
rates=[],
markup={}):
"""Create an OCP rate database object for test."""
table_name = OCP_REPORT_TABLE_MAP['cost_model']
cost_model_map = OCP_REPORT_TABLE_MAP['cost_model_map']
data = {
'uuid': str(uuid.uuid4()),
'created_timestamp': DateAccessor().today_with_timezone('UTC'),
'updated_timestamp': DateAccessor().today_with_timezone('UTC'),
'name': self.fake.pystr()[:8],
'description': self.fake.pystr(),
'source_type': source_type,
'rates': rates,
'markup': markup
}
with ProviderDBAccessor(provider_uuid) as accessor:
provider_obj = accessor.get_provider()
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
cost_model_obj = accessor.create_db_object(table_name, data)
data = {
'provider_uuid': provider_obj.uuid,
'cost_model_id': cost_model_obj.uuid
}
accessor.create_db_object(cost_model_map, data)
return cost_model_obj
def _get_sql_inputs(self, start_date, end_date):
"""Get the required inputs for running summary SQL."""
# Default to this month's bill
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
if self._manifest:
# Override the bill date to correspond with the manifest
bill_date = self._manifest.billing_period_start_datetime.date()
report_periods = accessor.get_usage_period_query_by_provider(
self._provider.id)
report_periods = report_periods.filter(
report_period_start=bill_date).all()
do_month_update = True
with schema_context(self._schema_name):
if report_periods is not None and len(report_periods) > 0:
do_month_update = self._determine_if_full_summary_update_needed(
report_periods[0])
if do_month_update:
last_day_of_month = calendar.monthrange(
bill_date.year, bill_date.month)[1]
start_date = bill_date.strftime('%Y-%m-%d')
end_date = bill_date.replace(day=last_day_of_month)
end_date = end_date.strftime('%Y-%m-%d')
LOG.info(
'Overriding start and end date to process full month.')
LOG.info('Returning start: %s, end: %s', str(start_date),
str(end_date))
return start_date, end_date
def create_ocpawscostlineitem_project_daily_summary(
self, account_id, schema):
"""Create an ocpawscostlineitem_project_daily_summary object for test."""
table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_project_daily_summary"]
data = self.create_columns_for_table(table_name)
with AccountAliasAccessor(account_id, schema) as accessor:
account_alias = accessor._get_db_obj_query().first()
data = {
"account_alias_id": account_alias.id,
"cost_entry_bill":
self.create_cost_entry_bill(str(uuid.uuid4())),
"namespace": self.fake.pystr()[:8],
"pod": self.fake.pystr()[:8],
"node": self.fake.pystr()[:8],
"usage_start":
self.make_datetime_aware(self.fake.past_datetime()),
"usage_end":
self.make_datetime_aware(self.fake.past_datetime()),
"product_code": self.fake.pystr()[:8],
"usage_account_id": self.fake.pystr()[:8],
}
with OCPReportDBAccessor(self.schema) as accessor:
return accessor.create_db_object(table_name, data)
def setUpClass(cls):
"""Set up the test class with required objects."""
# These test reports should be replaced with OCP reports once processor is impelmented.
cls.test_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
cls.storage_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv'
cls.unknown_report = './tests/data/test_cur.csv'
cls.test_report_gzip = './tests/data/test_cur.csv.gz'
cls.ocp_processor = OCPReportProcessor(schema_name='acct10001',
report_path=cls.test_report,
compression=UNCOMPRESSED,
provider_id=1)
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def _update_markup_cost(self, start_date, end_date):
"""Populate markup costs for OpenShift.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
infra_markup_value = Decimal(0.0)
infra_map = self.get_infra_map()
infra_tuple = infra_map.get(self._provider_uuid)
cluster_id = get_cluster_id_from_provider(self._provider_uuid)
if infra_tuple:
infra_uuid = infra_tuple[0]
with CostModelDBAccessor(self._schema, infra_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
infra_markup_value = Decimal(markup.get('value', 0)) / 100
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
ocp_markup_value = Decimal(markup.get('value', 0)) / 100
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info('Updating OpenShift markup for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema, self._provider_uuid, start_date, end_date)
accessor.populate_markup_cost(infra_markup_value, ocp_markup_value, cluster_id)
LOG.info('Finished updating markup.')
def create_cost_model(self,
provider_uuid,
source_type,
rates=[],
markup={}):
"""Create an OCP rate database object for test."""
table_name = OCP_REPORT_TABLE_MAP["cost_model"]
cost_model_map = OCP_REPORT_TABLE_MAP["cost_model_map"]
data = {
"uuid": str(uuid.uuid4()),
"created_timestamp": DateAccessor().today_with_timezone("UTC"),
"updated_timestamp": DateAccessor().today_with_timezone("UTC"),
"name": self.fake.pystr()[:8],
"description": self.fake.pystr(),
"source_type": source_type,
"rates": rates,
"markup": markup,
}
with ProviderDBAccessor(provider_uuid) as accessor:
provider_obj = accessor.get_provider()
with OCPReportDBAccessor(self.schema) as accessor:
cost_model_obj = accessor.create_db_object(table_name, data)
data = {
"provider_uuid": provider_obj.uuid,
"cost_model_id": cost_model_obj.uuid
}
accessor.create_db_object(cost_model_map, data)
return cost_model_obj
def process(self, parquet_base_filename, daily_data_frame):
"""Filter data and convert to parquet."""
for ocp_provider_uuid, infra_tuple in self.ocp_infrastructure_map.items(
):
infra_provider_uuid = infra_tuple[0]
if infra_provider_uuid != self.provider_uuid:
continue
msg = (
f"Processing OpenShift on {self.provider_type} to parquet."
f"\n\tStart date: {str(self.start_date)}\n\tFile: {str(self.report_file)}"
)
LOG.info(msg)
# Get OpenShift topology data
with OCPReportDBAccessor(self.schema_name) as accessor:
cluster_topology = accessor.get_openshift_topology_for_provider(
ocp_provider_uuid)
# Get matching tags
report_period_id = self.get_report_period_id(ocp_provider_uuid)
matched_tags = self.db_accessor.get_openshift_on_cloud_matched_tags(
self.bill_id, report_period_id)
if not matched_tags:
matched_tags = self.db_accessor.get_openshift_on_cloud_matched_tags_trino(
self.provider_uuid, ocp_provider_uuid, self.start_date,
self.end_date)
LOG.info(matched_tags)
openshift_filtered_data_frame = self.ocp_on_cloud_data_processor(
daily_data_frame, cluster_topology, matched_tags)
self.create_ocp_on_cloud_parquet(openshift_filtered_data_frame,
ocp_provider_uuid)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
report_periods = None
with OCPReportDBAccessor(self._schema) as accessor:
with schema_context(self._schema):
report_periods = accessor.report_periods_for_provider_uuid(
self._provider.uuid, start_date)
report_period_ids = [
report_period.id for report_period in report_periods
]
for report_period in report_periods:
LOG.info(
"Updating OpenShift report summary tables for \n\tSchema: %s "
"\n\tProvider: %s \n\tCluster: %s \n\tReport Period ID: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
self._cluster_id,
report_period.id,
start_date,
end_date,
)
# This will process POD and STORAGE together
accessor.populate_line_item_daily_summary_table_presto(
start_date, end_date, report_period.id, self._cluster_id,
self._cluster_alias, self._provider.uuid)
# This will process POD and STORAGE together
LOG.info(
"Updating OpenShift label summary tables for \n\tSchema: %s "
"\n\tReport Period IDs: %s",
self._schema,
report_period_ids,
)
accessor.populate_pod_label_summary_table(report_period_ids)
accessor.populate_volume_label_summary_table(report_period_ids)
accessor.update_line_item_daily_summary_with_enabled_tags(
start_date, end_date, report_period_ids)
LOG.info("Updating OpenShift report periods")
for period in report_periods:
if period.summary_data_creation_datetime is None:
period.summary_data_creation_datetime = self._date_accessor.today_with_timezone(
"UTC")
period.summary_data_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
period.save()
return start_date, end_date
def _get_sql_inputs(self, start_date, end_date):
"""Get the required inputs for running summary SQL."""
with OCPReportDBAccessor(self._schema) as accessor:
# This is the normal processing route
if self._manifest:
# Override the bill date to correspond with the manifest
bill_date = self._manifest.billing_period_start_datetime.date()
report_periods = accessor.get_usage_period_query_by_provider(
self._provider.uuid)
report_periods = report_periods.filter(
report_period_start=bill_date).all()
first_period = report_periods.first()
do_month_update = False
with schema_context(self._schema):
if first_period:
do_month_update = determine_if_full_summary_update_needed(
first_period)
if do_month_update:
last_day_of_month = calendar.monthrange(
bill_date.year, bill_date.month)[1]
start_date = bill_date
end_date = bill_date.replace(day=last_day_of_month)
LOG.info(
"Overriding start and end date to process full month.")
if isinstance(start_date, str):
start_date = ciso8601.parse_datetime(start_date).date()
if isinstance(end_date, str):
end_date = ciso8601.parse_datetime(end_date).date()
return start_date, end_date
def create_ocp_report_period(self,
period_date=None,
provider_id=None,
cluster_id=None):
"""Create an OCP report database object for test."""
table_name = OCP_REPORT_TABLE_MAP['report_period']
period_start = self.make_datetime_aware(
self.fake.past_datetime()).date().replace(day=1)
period_end = period_start + relativedelta.relativedelta(
days=random.randint(1, 15))
data = {
'cluster_id': cluster_id if cluster_id else self.fake.pystr()[:8],
'provider_id': provider_id if provider_id else 1,
'report_period_start': period_start,
'report_period_end': period_end,
}
if period_date:
period_start = period_date.replace(day=1).date()
period_end = period_start + relativedelta.relativedelta(months=1)
data['report_period_start'] = period_start
data['report_period_end'] = period_end
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def test_update_summary_markup_charge_no_clusterid(self):
"""Test that markup is calculated without a cluster id."""
markup = {'value': 10, 'unit': 'percent'}
markup_percentage = Decimal(markup.get('value')) / 100
rate = [
{
'metric': {'name': 'memory_gb_usage_per_hour'},
'tiered_rates': [{'value': 1, 'unit': 'USD'}],
}
]
self.creator.create_cost_model(self.ocp_provider_uuid, Provider.PROVIDER_OCP, rates=rate, markup=markup)
usage_period = self.accessor.get_current_usage_period()
start_date = usage_period.report_period_start.date() + relativedelta(days=-1)
end_date = usage_period.report_period_end.date() + relativedelta(days=+1)
self.accessor.populate_line_item_daily_table(start_date, end_date, self.cluster_id)
self.accessor.populate_line_item_daily_summary_table(start_date, end_date, self.cluster_id)
self.updater._update_pod_charge(start_date, end_date)
self.updater._update_storage_charge(start_date, end_date)
self.updater._cluster_id = None
self.updater._update_markup_cost(start_date, end_date)
table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']
with OCPReportDBAccessor(schema=self.schema, column_map=self.column_map) as accessor:
with schema_context(self.schema):
items = accessor._get_db_obj_query(table_name).all()
for item in items:
markup_value = Decimal(item.markup_cost)
self.assertEqual(
markup_value, item.pod_charge_memory_gigabyte_hours * markup_percentage
)
def create_ocp_usage_line_item(self,
report_period,
report,
resource_id=None,
pod=None,
namespace=None,
null_cpu_usage=False):
"""Create an OCP usage line item database object for test."""
table_name = OCP_REPORT_TABLE_MAP['line_item']
data = self.create_columns_for_table(table_name)
for key in data:
if 'byte' in key:
data[key] = data[key] * Decimal(pow(2, 30))
if resource_id:
data['resource_id'] = resource_id
data['report_period_id'] = report_period.id
data['report_id'] = report.id
if null_cpu_usage:
data['pod_usage_cpu_core_seconds'] = None
if pod:
data['pod'] = pod
if namespace:
data['namespace'] = namespace
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def _update_markup_cost(self, start_date, end_date):
"""Populate markup costs for OpenShift.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
with CostModelDBAccessor(self._schema, self._provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup = Decimal(markup.get("value", 0)) / 100
with OCPReportDBAccessor(self._schema) as accessor:
LOG.info(
"Updating markup for" "\n\tSchema: %s \n\t%s Provider: %s (%s) \n\tDates: %s - %s",
self._schema,
self._provider.type,
self._provider.name,
self._provider_uuid,
start_date,
end_date,
)
accessor.populate_markup_cost(markup, start_date, end_date, self._cluster_id)
LOG.info("Finished updating markup.")
def test_update_summary_tables(
self,
mock_sum,
mock_tag_sum,
mock_vol_tag_sum,
mock_delete,
mock_cluster_populate,
mock_date_check,
mock_infra_check,
):
"""Test that summary tables are run for a full month when no report period is found."""
start_date = self.dh.today
end_date = start_date
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
mock_date_check.return_value = (start_date, end_date)
with OCPReportDBAccessor(self.schema) as accessor:
with schema_context(self.schema):
report_period = accessor.report_periods_for_provider_uuid(
self.provider.uuid, start_date)
report_period_id = report_period.id
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_delete.assert_called_with(self.ocp_provider.uuid,
start_date.date(), end_date.date(),
{"report_period_id": report_period_id})
mock_sum.assert_called()
mock_tag_sum.assert_called()
mock_vol_tag_sum.assert_called()
mock_date_check.assert_called()
mock_infra_check.assert_called()
def update_summary_charge_info(self, start_date=None, end_date=None):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
LOG.info(
'Starting charge calculation updates for provider: %s. Cluster ID: %s.',
self._provider_uuid, self._cluster_id)
self._update_pod_charge()
self._update_storage_charge()
self._update_markup_cost()
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
report_periods = accessor.report_periods_for_provider_uuid(
self._provider_uuid, start_date)
with schema_context(self._schema):
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone('UTC')
period.save()
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.accessor, cls.column_map,
cls.report_schema.column_types)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
day=1)
cls.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'num_processed_files': 2,
'provider_id': 2
}
cls.manifest_accessor = ReportManifestDBAccessor()
def update_daily_tables(self, start_date, end_date):
"""Populate the daily tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating OpenShift report daily tables for \n\tSchema: %s "
"\n\tProvider: %s \n\tCluster: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
self._cluster_id,
start,
end,
)
with OCPReportDBAccessor(self._schema) as accessor:
accessor.populate_node_label_line_item_daily_table(start, end, self._cluster_id)
accessor.populate_line_item_daily_table(start, end, self._cluster_id)
accessor.populate_storage_line_item_daily_table(start, end, self._cluster_id)
return start_date, end_date
def _update_storage_charge(self):
"""Calculate and store the storage charges."""
try:
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
storage_usage_rates = cost_model_accessor.get_storage_gb_usage_per_month_rates(
)
storage_request_rates = cost_model_accessor.get_storage_gb_request_per_month_rates(
)
with OCPReportDBAccessor(self._schema,
self._column_map) as report_accessor:
storage_usage = report_accessor.\
get_persistentvolumeclaim_usage_gigabyte_months(self._cluster_id)
storage_usage_charge = self._calculate_charge(
storage_usage_rates, storage_usage)
storage_request = report_accessor.\
get_volume_request_storage_gigabyte_months(self._cluster_id)
storage_request_charge = self._calculate_charge(
storage_request_rates, storage_request)
total_storage_charge = self._aggregate_charges(
storage_usage_charge, storage_request_charge)
temp_table = self._write_to_temp_table(report_accessor,
total_storage_charge)
report_accessor.populate_storage_charge(temp_table)
except OCPReportChargeUpdaterError as error:
LOG.error('Unable to calculate storage usage charge. Error: %s',
str(error))
def _update_usage_costs(self, start_date, end_date):
"""Update infrastructure and supplementary usage costs."""
with OCPReportDBAccessor(self._schema) as report_accessor:
report_accessor.populate_usage_costs(self._infra_rates,
self._supplementary_rates,
start_date, end_date,
self._cluster_id)
def test_create_usage_report_line_item_storage_missing_labels(self):
"""Test that line item data is returned properly."""
cluster_id = "12345"
storage_processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.storage_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
with OCPReportDBAccessor(self.schema) as accessor:
report_period_id = storage_processor._processor._create_report_period(
self.storage_row, cluster_id, accessor, self.cluster_alias)
report_id = storage_processor._processor._create_report(
self.storage_row, report_period_id, accessor)
row = copy.deepcopy(self.storage_row)
del row["persistentvolume_labels"]
del row["persistentvolumeclaim_labels"]
storage_processor._processor._create_usage_report_line_item(
row, report_period_id, report_id, accessor)
line_item = None
if storage_processor._processor.processed_report.line_items:
line_item = storage_processor._processor.processed_report.line_items[
-1]
self.assertIsNotNone(line_item)
self.assertEqual(line_item.get("report_period_id"), report_period_id)
self.assertEqual(line_item.get("report_id"), report_id)
self.assertEqual(line_item.get("persistentvolume_labels"), "{}")
self.assertEqual(line_item.get("persistentvolumeclaim_labels"), "{}")
self.assertIsNotNone(storage_processor._processor.line_item_columns)
def update_summary_cost_model_costs(self, start_date, end_date):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
if isinstance(start_date, str):
start_date = parse(start_date)
if isinstance(end_date, str):
end_date = parse(end_date)
LOG.info(
"Updating cost model costs for \n%s provider: %s (%s). \nCluster ID: %s.",
self._provider.type,
self._provider.name,
self._provider_uuid,
self._cluster_id,
)
self._update_usage_costs(start_date, end_date)
self._update_markup_cost(start_date, end_date)
self._update_monthly_cost(start_date, end_date)
with OCPReportDBAccessor(self._schema) as accessor:
report_periods = accessor.report_periods_for_provider_uuid(
self._provider_uuid, start_date)
with schema_context(self._schema):
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone("UTC")
period.save()
def test_get_report_period_id(self):
"""Test that the OpenShift cluster's report period ID is returned."""
with OCPReportDBAccessor(self.schema_name) as accessor:
with schema_context(self.schema_name):
report_period = accessor.report_periods_for_provider_uuid(self.ocp_provider_uuid, self.start_date)
expected = report_period.id
self.assertEqual(self.report_processor.get_report_period_id(self.ocp_provider_uuid), expected)
def setUp(self):
"""Set up each test."""
super().setUp()
self.aws_accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map)
self.ocp_accessor = OCPReportDBAccessor(schema=self.schema, column_map=self.column_map)
# Populate some line item data so that the summary tables
# have something to pull from
self.start_date = DateAccessor().today_with_timezone("UTC").replace(day=1)
last_month = self.start_date - relativedelta.relativedelta(months=1)
for cost_entry_date in (self.start_date, last_month):
bill = self.creator.create_cost_entry_bill(provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date)
cost_entry = self.creator.create_cost_entry(bill, cost_entry_date)
for family in ["Storage", "Compute Instance", "Database Storage", "Database Instance"]:
product = self.creator.create_cost_entry_product(family)
pricing = self.creator.create_cost_entry_pricing()
reservation = self.creator.create_cost_entry_reservation()
self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation)
provider_ocp_uuid = self.ocp_test_provider_uuid
with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor:
provider_uuid = provider_accessor.get_provider().uuid
cluster_id = self.ocp_provider_resource_name
for period_date in (self.start_date, last_month):
period = self.creator.create_ocp_report_period(
provider_uuid=provider_uuid, period_date=period_date, cluster_id=cluster_id
)
report = self.creator.create_ocp_report(period, period_date)
for _ in range(25):
self.creator.create_ocp_usage_line_item(period, report)
