def test_get_bill_ids_from_provider(self):
"""Test that bill IDs are returned for an AWS provider."""
with schema_context(self.schema):
expected_bill_ids = AWSCostEntryBill.objects.values_list("id")
expected_bill_ids = sorted([bill_id[0] for bill_id in expected_bill_ids])
bills = utils.get_bills_from_provider(self.aws_provider_uuid, self.schema)
with schema_context(self.schema):
bill_ids = sorted([bill.id for bill in bills])
self.assertEqual(bill_ids, expected_bill_ids)
# Try with unknown provider uuid
bills = utils.get_bills_from_provider(self.unkown_test_provider_uuid, self.schema)
self.assertEqual(bills, [])
def test_get_bill_ids_from_provider_with_start_and_end_date(self):
"""Test that bill IDs are returned for an AWS provider with both dates."""
date_accessor = DateAccessor()
with ProviderDBAccessor(provider_uuid=self.aws_test_provider_uuid
) as provider_accessor:
provider = provider_accessor.get_provider()
with AWSReportDBAccessor(schema=self.test_schema,
column_map=self.column_map) as accessor:
report_schema = accessor.report_schema
creator = ReportObjectCreator(accessor, self.column_map,
report_schema.column_types)
end_date = date_accessor.today_with_timezone('utc').replace(day=1)
start_date = end_date
for i in range(2):
start_date = start_date - relativedelta(months=i)
print(start_date)
bill = creator.create_cost_entry_bill(bill_date=start_date)
bill_table_name = AWS_CUR_TABLE_MAP['bill']
bill_obj = getattr(accessor.report_schema, bill_table_name)
bills = accessor.get_cost_entry_bills_query_by_provider(
provider.id)
bills = bills.filter(bill_obj.billing_period_start>=start_date.date())\
.filter(bill_obj.billing_period_start<=end_date.date()).all()
expected_bill_ids = [str(bill.id) for bill in bills]
bills = utils.get_bills_from_provider(self.aws_test_provider_uuid,
self.test_schema,
start_date=start_date,
end_date=end_date)
bill_ids = [str(bill.id) for bill in bills]
self.assertEqual(bill_ids, expected_bill_ids)
def update_daily_tables(self, start_date, end_date):
"""Populate the daily tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str): A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema,
datetime.datetime.strptime(start_date, "%Y-%m-%d"),
datetime.datetime.strptime(end_date, "%Y-%m-%d"),
)
bill_ids = []
with schema_context(self._schema):
bill_ids = [str(bill.id) for bill in bills]
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating AWS report daily tables for \n\tSchema: %s"
"\n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
start,
end,
)
accessor.populate_line_item_daily_table(start, end, bill_ids)
return start_date, end_date
def test_get_bill_ids_from_provider_with_start_and_end_date(self):
"""Test that bill IDs are returned for an AWS provider with both dates."""
date_accessor = DateAccessor()
with ProviderDBAccessor(
provider_uuid=self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with AWSReportDBAccessor(schema=self.schema) as accessor:
end_date = date_accessor.today_with_timezone("utc").replace(day=1)
start_date = end_date
for i in range(2):
start_date = start_date - relativedelta(months=i)
bills = accessor.get_cost_entry_bills_query_by_provider(
provider.uuid)
with schema_context(self.schema):
bills = (bills.filter(
billing_period_start__gte=start_date.date()).filter(
billing_period_start__lte=end_date.date()).all())
expected_bill_ids = [str(bill.id) for bill in bills]
bills = utils.get_bills_from_provider(self.aws_provider_uuid,
self.schema,
start_date=start_date,
end_date=end_date)
with schema_context(self.schema):
bill_ids = [str(bill.id) for bill in bills]
self.assertEqual(bill_ids, expected_bill_ids)
def update_daily_tables(self, start_date, end_date):
"""Populate the daily tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str): A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema_name,
start_date,
end_date
)
bill_ids = [str(bill.id) for bill in bills]
LOG.info('Updating AWS report daily tables for \n\tSchema: %s'
'\n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor:
accessor.populate_line_item_daily_table(start_date, end_date, bill_ids)
return start_date, end_date
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
infra_map = self._get_ocp_infra_map(start_date, end_date)
cluster_id = self._get_ocp_cluster_id_for_provider(
self._provider, start_date, end_date, infra_map)
if cluster_id:
aws_uuid = self._get_aws_provider_uuid_from_map(
self._provider, infra_map)
aws_bills = get_bills_from_provider(aws_uuid, self._schema_name,
start_date, end_date)
aws_bill_ids = [str(bill.id) for bill in aws_bills]
# OpenShift on AWS
with AWSReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s'
'\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema_name,
self._provider.uuid, start_date, end_date, cluster_id,
str(aws_bill_ids))
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.commit()
else:
LOG.info(
'Provider: %s is not part of an OCP-on-AWS configuration.',
self._provider.name)
# This needs to always run regardless of whether the OpenShift
# cluster is tied to a cloud provider
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
accessor.commit()
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema,
datetime.datetime.strptime(start_date, "%Y-%m-%d"),
datetime.datetime.strptime(end_date, "%Y-%m-%d"),
)
bill_ids = []
with schema_context(self._schema):
bill_ids = [str(bill.id) for bill in bills]
with AWSReportDBAccessor(self._schema) as accessor:
# Need these bills on the session to update dates after processing
bills = accessor.bills_for_provider_uuid(self._provider.uuid,
start_date)
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating AWS report summary tables: \n\tSchema: %s"
"\n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
start,
end,
)
accessor.populate_line_item_daily_summary_table(
start, end, bill_ids)
accessor.populate_tags_summary_table(bill_ids, start_date,
end_date)
for bill in bills:
if bill.summary_data_creation_datetime is None:
bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.save()
return start_date, end_date
def _update_markup_cost(self, start_date, end_date):
"""Store markup costs."""
try:
bills = get_bills_from_provider(self._provider.uuid, self._schema,
start_date, end_date)
with CostModelDBAccessor(
self._schema, self._provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
with AWSReportDBAccessor(self._schema) as report_accessor:
with schema_context(self._schema):
bill_ids = [str(bill.id) for bill in bills]
report_accessor.populate_markup_cost(markup_value, bill_ids)
except AWSCostModelCostUpdaterError as error:
LOG.error("Unable to update markup costs. Error: %s", str(error))
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
infra_map = self._get_ocp_infra_map(start_date, end_date)
cluster_id = self._get_ocp_cluster_id_for_provider(
self._provider, start_date, end_date, infra_map)
if cluster_id:
aws_uuid = self._get_aws_provider_uuid_from_map(
self._provider, infra_map)
aws_bills = get_bills_from_provider(
aws_uuid, self._schema_name,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d'))
aws_bill_ids = []
with schema_context(self._schema_name):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
with CostModelDBAccessor(self._schema_name, aws_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = float(markup.get('value', 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s'
'\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema_name,
self._provider.uuid, start_date, end_date, cluster_id,
str(aws_bill_ids))
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.populate_ocp_on_aws_markup_cost(
markup_value, aws_bill_ids)
else:
LOG.info(
'Provider: %s is not part of an OCP-on-AWS configuration.',
self._provider.name)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid, self._schema_name,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d'))
bill_ids = []
with schema_context(self._schema_name):
bill_ids = [str(bill.id) for bill in bills]
with AWSReportDBAccessor(self._schema_name,
self._column_map) as accessor:
# Need these bills on the session to update dates after processing
bills = accessor.bills_for_provider_id(self._provider.id,
start_date)
LOG.info(
'Updating AWS report summary tables: \n\tSchema: %s'
'\n\tProvider: %s \n\tDates: %s - %s', self._schema_name,
self._provider.uuid, start_date, end_date)
accessor.populate_line_item_daily_summary_table(
start_date, end_date, bill_ids)
accessor.populate_tags_summary_table()
for bill in bills:
if bill.summary_data_creation_datetime is None:
bill.summary_data_creation_datetime = \
self._date_accessor.today_with_timezone('UTC')
bill.summary_data_updated_datetime = \
self._date_accessor.today_with_timezone('UTC')
bill.save()
accessor.commit()
return start_date, end_date
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
aws_bills = get_bills_from_provider(self._provider.uuid,
self._schema_name, start_date,
end_date)
aws_bill_ids = [str(bill.id) for bill in aws_bills]
# OpenShift on AWS
with AWSReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.commit()
if cluster_id:
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date,
end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
accessor.commit()
def test_get_bill_ids_from_provider(self):
"""Test that bill IDs are returned for an AWS provider."""
date_accessor = DateAccessor()
with AWSReportDBAccessor(schema=self.test_schema,
column_map=self.column_map) as accessor:
report_schema = accessor.report_schema
creator = ReportObjectCreator(accessor, self.column_map,
report_schema.column_types)
expected_bill_ids = []
end_date = date_accessor.today_with_timezone('utc').replace(day=1)
start_date = end_date
for i in range(2):
start_date = start_date - relativedelta(months=i)
bill = creator.create_cost_entry_bill(bill_date=start_date)
expected_bill_ids.append(str(bill.id))
bills = utils.get_bills_from_provider(self.aws_test_provider_uuid,
self.test_schema)
bill_ids = [str(bill.id) for bill in bills]
self.assertEqual(sorted(bill_ids), sorted(expected_bill_ids))
def test_get_bill_ids_from_provider(self):
"""Test that bill IDs are returned for an AWS provider."""
date_accessor = DateAccessor()
creator = ReportObjectCreator(self.schema, self.column_map)
expected_bill_ids = []
end_date = date_accessor.today_with_timezone("utc").replace(day=1)
start_date = end_date
for i in range(2):
start_date = start_date - relativedelta(months=i)
bill = creator.create_cost_entry_bill(
provider_uuid=self.aws_provider_uuid, bill_date=start_date)
with schema_context(self.schema):
expected_bill_ids.append(str(bill.id))
bills = utils.get_bills_from_provider(self.aws_provider_uuid,
self.schema)
with schema_context(self.schema):
bill_ids = [str(bill.id) for bill in bills]
self.assertEqual(sorted(bill_ids), sorted(expected_bill_ids))