def setUp(self):
super().setUp()
self.accessor = AWSReportDBAccessor('acct10001', self.column_map)
self.report_schema = self.accessor.report_schema
self.session = self.accessor._session
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
def test_update_summary_tables(self):
"""Test that summary tables are updated correctly."""
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_daily_summary']
query = aws_accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
updater.update_summary_tables(start_date_str, end_date_str)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
query = aws_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
def setUp(self):
"""Setup tests."""
super().setUp()
self.column_map = ReportingCommonDBAccessor().column_map
self.accessor = AWSReportDBAccessor(schema='acct10001',
column_map=self.column_map)
self.manifest_accessor = ReportManifestDBAccessor()
def test_update_summary_tables_with_manifest(self, mock_daily,
mock_summary):
"""Test that summary tables are properly run."""
self.manifest.num_processed_files = self.manifest.num_total_files
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with AWSReportDBAccessor(self.schema) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.summary_data_creation_datetime = start_date
bill.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = start_date.date()
expected_end_date = end_date.date()
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
with AWSReportDBAccessor(self.schema) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)
def setUp(self):
"""Set up each test."""
super().setUp()
self.aws_accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map)
self.ocp_accessor = OCPReportDBAccessor(schema=self.schema, column_map=self.column_map)
# Populate some line item data so that the summary tables
# have something to pull from
self.start_date = DateAccessor().today_with_timezone("UTC").replace(day=1)
last_month = self.start_date - relativedelta.relativedelta(months=1)
for cost_entry_date in (self.start_date, last_month):
bill = self.creator.create_cost_entry_bill(provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date)
cost_entry = self.creator.create_cost_entry(bill, cost_entry_date)
for family in ["Storage", "Compute Instance", "Database Storage", "Database Instance"]:
product = self.creator.create_cost_entry_product(family)
pricing = self.creator.create_cost_entry_pricing()
reservation = self.creator.create_cost_entry_reservation()
self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation)
provider_ocp_uuid = self.ocp_test_provider_uuid
with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor:
provider_uuid = provider_accessor.get_provider().uuid
cluster_id = self.ocp_provider_resource_name
for period_date in (self.start_date, last_month):
period = self.creator.create_ocp_report_period(
provider_uuid=provider_uuid, period_date=period_date, cluster_id=cluster_id
)
report = self.creator.create_ocp_report(period, period_date)
for _ in range(25):
self.creator.create_ocp_usage_line_item(period, report)
def setUp(self):
"""Set up shared variables."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f'{self.temp_dir}/test_cur.csv'
self.test_report_gzip = f'{self.temp_dir}/test_cur.csv.gz'
shutil.copy2(self.test_report_test_path, self.test_report)
shutil.copy2(self.test_report_gzip_test_path, self.test_report_gzip)
self.processor = AWSReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0
)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_uuid': self.aws_provider_uuid,
}
self.accessor = AWSReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def setUp(self):
"""Set up each test."""
super().setUp()
self.aws_accessor = AWSReportDBAccessor(schema=self.schema)
self.ocp_accessor = OCPReportDBAccessor(schema=self.schema)
# Populate some line item data so that the summary tables
# have something to pull from
self.start_date = DateHelper().today.replace(day=1)
def setUp(self):
"""Set up the test class with required objects."""
super().setUp()
self.accessor = AWSReportDBAccessor(schema=self.schema)
self.all_tables = list(AWS_CUR_TABLE_MAP.values())
self.foreign_key_tables = [
AWS_CUR_TABLE_MAP["bill"],
AWS_CUR_TABLE_MAP["product"],
AWS_CUR_TABLE_MAP["pricing"],
AWS_CUR_TABLE_MAP["reservation"],
]
def setUp(self):
"""Set up the test class with required objects."""
super().setUp()
self.common_accessor = ReportingCommonDBAccessor()
self.column_map = self.common_accessor.column_map
self.accessor = AWSReportDBAccessor(schema=self.schema,
column_map=self.column_map)
self.all_tables = list(AWS_CUR_TABLE_MAP.values())
self.foreign_key_tables = [
AWS_CUR_TABLE_MAP['bill'],
AWS_CUR_TABLE_MAP['product'],
AWS_CUR_TABLE_MAP['pricing'],
AWS_CUR_TABLE_MAP['reservation'],
]
def test_update_daily_summary_tables(self, mock_presto, mock_tag_update,
mock_delete):
"""Test that we run Presto summary."""
start_str = self.dh.this_month_start.isoformat()
end_str = self.dh.this_month_end.isoformat()
start, end = self.updater._get_sql_inputs(start_str, end_str)
for s, e in date_range_pair(start, end, step=settings.TRINO_DATE_STEP):
expected_start, expected_end = s, e
with AWSReportDBAccessor(self.schema) as accessor:
with schema_context(self.schema):
bills = accessor.bills_for_provider_uuid(
self.aws_provider.uuid, start)
bill_ids = [str(bill.id) for bill in bills]
current_bill_id = bills.first().id if bills else None
with CostModelDBAccessor(
self.schema, self.aws_provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
start_return, end_return = self.updater.update_summary_tables(
start, end)
mock_delete.assert_called_with(self.aws_provider.uuid, expected_start,
expected_end)
mock_presto.assert_called_with(expected_start, expected_end,
self.aws_provider.uuid, current_bill_id,
markup_value)
mock_tag_update.assert_called_with(bill_ids, start, end)
self.assertEqual(start_return, start)
self.assertEqual(end_return, end)
def _delete_line_items(self):
"""Delete stale data for the report being processed, if necessary."""
if not self.manifest_id:
return False
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest_by_id(self.manifest_id)
if manifest.num_processed_files != 0:
return False
# Override the bill date to correspond with the manifest
bill_date = manifest.billing_period_start_datetime.date()
provider_id = manifest.provider_id
LOG.info('Deleting data for schema: %s and bill date: %s',
self._schema_name, str(bill_date))
with AWSReportDBAccessor(self._schema_name,
self.column_map) as accessor:
bills = accessor.get_cost_entry_bills_query_by_provider(
provider_id)
bills = bills.filter_by(billing_period_start=bill_date).all()
for bill in bills:
line_item_query = accessor.get_lineitem_query_for_billid(
bill.id)
line_item_query.delete()
accessor.commit()
return True
def update_daily_tables(self, start_date, end_date):
"""Populate the daily tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str): A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema_name,
start_date,
end_date
)
bill_ids = [str(bill.id) for bill in bills]
LOG.info('Updating AWS report daily tables for \n\tSchema: %s'
'\n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor:
accessor.populate_line_item_daily_table(start_date, end_date, bill_ids)
return start_date, end_date
def test_update_summary_tables_without_manifest(self, mock_daily,
mock_summary):
"""Test that summary tables are properly run without a manifest."""
self.updater = AWSReportSummaryUpdater(self.schema, self.provider,
None)
start_date = datetime.datetime(year=self.today.year,
month=self.today.month,
day=self.today.day)
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.summary_data_updated_datetime = start_date
bill.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = start_date.date()
expected_end_date = end_date.date()
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
with AWSReportDBAccessor(self.schema) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertGreater(bill.summary_data_updated_datetime, self.today)
def test_get_bill_ids_from_provider_with_start_and_end_date(self):
"""Test that bill IDs are returned for an AWS provider with both dates."""
date_accessor = DateAccessor()
with ProviderDBAccessor(provider_uuid=self.aws_test_provider_uuid
) as provider_accessor:
provider = provider_accessor.get_provider()
with AWSReportDBAccessor(schema=self.test_schema,
column_map=self.column_map) as accessor:
report_schema = accessor.report_schema
creator = ReportObjectCreator(accessor, self.column_map,
report_schema.column_types)
end_date = date_accessor.today_with_timezone('utc').replace(day=1)
start_date = end_date
for i in range(2):
start_date = start_date - relativedelta(months=i)
print(start_date)
bill = creator.create_cost_entry_bill(bill_date=start_date)
bill_table_name = AWS_CUR_TABLE_MAP['bill']
bill_obj = getattr(accessor.report_schema, bill_table_name)
bills = accessor.get_cost_entry_bills_query_by_provider(
provider.id)
bills = bills.filter(bill_obj.billing_period_start>=start_date.date())\
.filter(bill_obj.billing_period_start<=end_date.date()).all()
expected_bill_ids = [str(bill.id) for bill in bills]
bills = utils.get_bills_from_provider(self.aws_test_provider_uuid,
self.test_schema,
start_date=start_date,
end_date=end_date)
bill_ids = [str(bill.id) for bill in bills]
self.assertEqual(bill_ids, expected_bill_ids)
def db_accessor(self):
"""Return the accessor for the infrastructure provider."""
if self.provider_type == Provider.PROVIDER_AWS:
return AWSReportDBAccessor(self.schema_name)
elif self.provider_type == Provider.PROVIDER_AZURE:
return AzureReportDBAccessor(self.schema_name)
return None
def tearDown(self):
"""Return the database to a pre-test state."""
self.accessor._session.rollback()
aws_tables = list(AWS_CUR_TABLE_MAP.values())
with AWSReportDBAccessor(self.test_schema,
self.column_map) as aws_accessor:
aws_accessor._session.rollback()
for table_name in aws_tables:
tables = aws_accessor._get_db_obj_query(table_name).all()
for table in tables:
aws_accessor._session.delete(table)
aws_accessor.commit()
ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
with OCPReportDBAccessor(self.test_schema,
self.column_map) as ocp_accessor:
for table_name in ocp_tables:
tables = ocp_accessor._get_db_obj_query(table_name).all()
for table in tables:
ocp_accessor._session.delete(table)
ocp_accessor.commit()
manifests = self.manifest_accessor._get_db_obj_query().all()
for manifest in manifests:
self.manifest_accessor.delete(manifest)
self.manifest_accessor.commit()
def test_update_summary_tables_finalized_bill_not_done_proc(
self, mock_daily, mock_summary):
"""Test that summary tables are run for a full month."""
report_updater_base = ReportSummaryUpdater('acct10001',
self.aws_provider_uuid,
self.manifest.id)
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.finalized_datetime = start_date
self.accessor.commit()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
self.assertIsNone(bill.summary_data_creation_datetime)
self.assertIsNone(bill.summary_data_updated_datetime)
if report_updater_base.manifest_is_ready():
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called()
if report_updater_base.manifest_is_ready():
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called()
with AWSReportDBAccessor('acct10001', self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = AWSReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
cls.all_tables = list(AWS_CUR_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.accessor, cls.column_map,
cls.report_schema.column_types)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
day=1)
cls.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': 1,
}
cls.manifest_accessor = ReportManifestDBAccessor()
def setUpClass(cls):
"""Set up the test class with required objects."""
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.accessor = AWSReportDBAccessor(
schema='acct10001',
column_map=cls.column_map
)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(
cls.accessor,
cls.column_map,
cls.report_schema.column_types
)
cls.all_tables = list(AWS_CUR_TABLE_MAP.values())
cls.foreign_key_tables = [
AWS_CUR_TABLE_MAP['bill'],
AWS_CUR_TABLE_MAP['product'],
AWS_CUR_TABLE_MAP['pricing'],
AWS_CUR_TABLE_MAP['reservation']
]
billing_start = datetime.datetime.utcnow().replace(day=1)
cls.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': 1
}
cls.manifest_accessor = ReportManifestDBAccessor()
def test_update_summary_tables_new_bill_not_done_processing(
self, mock_daily, mock_summary):
"""Test that summary tables are not run for a full month."""
report_updater_base = ReportSummaryUpdater("acct10001",
self.aws_provider_uuid,
self.manifest.id)
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
self.assertIsNone(bill.summary_data_creation_datetime)
self.assertIsNone(bill.summary_data_updated_datetime)
# manifest_is_ready is now unconditionally returning True, so summary is expected.
if report_updater_base.manifest_is_ready():
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called()
if report_updater_base.manifest_is_ready():
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called()
with AWSReportDBAccessor("acct10001", self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)
def test_update_markup_cost(self, mock_cost_model):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
markup_dec = decimal.Decimal(markup.get("value") / 100)
mock_cost_model.markup = markup
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
manifest = CostUsageReportManifest.objects.filter(
provider=self.ocp_on_aws_ocp_provider,
billing_period_start_datetime=start_date).first()
updater = OCPCloudReportSummaryUpdater(
schema=self.schema,
provider=self.ocp_on_aws_ocp_provider,
manifest=manifest)
updater.update_summary_tables(start_date, end_date)
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
with AWSReportDBAccessor(self.schema) as aws_accessor:
query = (aws_accessor._get_db_obj_query(summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date).all())
for item in query:
self.assertAlmostEqual(item.markup_cost,
item.unblended_cost * markup_dec)
def test_update_project_markup_cost(self, mock_cost_model):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_cost_model.return_value.__enter__.return_value.markup = markup
markup_dec = decimal.Decimal(markup.get("value") / 100)
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
updater = OCPCloudReportSummaryUpdater(
schema=self.schema,
provider=self.ocp_on_aws_ocp_provider,
manifest=None)
updater.update_summary_tables(start_date, end_date,
self.ocp_on_aws_ocp_provider.uuid,
self.aws_provider.uuid,
Provider.PROVIDER_AWS)
summary_table_name = AWS_CUR_TABLE_MAP[
"ocp_on_aws_project_daily_summary"]
with AWSReportDBAccessor(self.schema) as aws_accessor:
query = (aws_accessor._get_db_obj_query(summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date,
data_source="Pod").all())
for item in query:
self.assertAlmostEqual(item.project_markup_cost,
item.pod_cost * markup_dec)
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
aws_bills = aws_get_bills_from_provider(
aws_provider_uuid, self._schema,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d'))
aws_bill_ids = []
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
with CostModelDBAccessor(self._schema, aws_provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = Decimal(markup.get('value', 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s'
'\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema,
self._provider.uuid, start_date, end_date, cluster_id,
str(aws_bill_ids))
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.populate_ocp_on_aws_markup_cost(markup_value,
aws_bill_ids)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def test_update_summary_tables_without_manifest(self, mock_daily,
mock_summary):
"""Test that summary tables are properly run without a manifest."""
self.updater = AWSReportSummaryUpdater('acct10001', self.provider,
None)
start_date = DateAccessor().today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.summary_data_updated_datetime = start_date
self.accessor.commit()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
with AWSReportDBAccessor('acct10001', self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertGreater(bill.summary_data_updated_datetime, start_date)
def create_cost_entry_line_item(self,
bill,
cost_entry,
product,
pricing,
reservation,
resource_id=None):
"""Create a cost entry line item database object for test."""
table_name = AWS_CUR_TABLE_MAP["line_item"]
data = self.create_columns_for_table(table_name)
extra_data = {
"cost_entry_bill_id": bill.id,
"cost_entry_id": cost_entry.id,
"cost_entry_product_id": product.id,
"cost_entry_pricing_id": pricing.id,
"cost_entry_reservation_id": reservation.id,
"usage_start": cost_entry.interval_start,
"usage_end": cost_entry.interval_end,
"usage_account_id": self.fake.pystr()[:8],
"resource_id": resource_id,
"tags": {
"environment": random.choice(["dev", "qa", "prod"]),
self.fake.pystr()[:8]: self.fake.pystr()[:8],
},
}
data.update(extra_data)
with AWSReportDBAccessor(self.schema) as accessor:
return accessor.create_db_object(table_name, data)
def test_get_bill_ids_from_provider_with_start_and_end_date(self):
"""Test that bill IDs are returned for an AWS provider with both dates."""
date_accessor = DateAccessor()
with ProviderDBAccessor(
provider_uuid=self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with AWSReportDBAccessor(schema=self.schema) as accessor:
end_date = date_accessor.today_with_timezone("utc").replace(day=1)
start_date = end_date
for i in range(2):
start_date = start_date - relativedelta(months=i)
bills = accessor.get_cost_entry_bills_query_by_provider(
provider.uuid)
with schema_context(self.schema):
bills = (bills.filter(
billing_period_start__gte=start_date.date()).filter(
billing_period_start__lte=end_date.date()).all())
expected_bill_ids = [str(bill.id) for bill in bills]
bills = utils.get_bills_from_provider(self.aws_provider_uuid,
self.schema,
start_date=start_date,
end_date=end_date)
with schema_context(self.schema):
bill_ids = [str(bill.id) for bill in bills]
self.assertEqual(bill_ids, expected_bill_ids)
def create_cost_entry_line_item(self,
bill,
cost_entry,
product,
pricing,
reservation,
resource_id=None):
"""Create a cost entry line item database object for test."""
table_name = AWS_CUR_TABLE_MAP['line_item']
data = self.create_columns_for_table(table_name)
extra_data = {
'cost_entry_bill_id': bill.id,
'cost_entry_id': cost_entry.id,
'cost_entry_product_id': product.id,
'cost_entry_pricing_id': pricing.id,
'cost_entry_reservation_id': reservation.id,
'usage_start': cost_entry.interval_start,
'usage_end': cost_entry.interval_end,
'resource_id': resource_id,
'tags': {
'environment': random.choice(['dev', 'qa', 'prod']),
self.fake.pystr()[:8]: self.fake.pystr()[:8],
}
}
data.update(extra_data)
with AWSReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def _get_sql_inputs(self, start_date, end_date):
"""Get the required inputs for running summary SQL."""
with AWSReportDBAccessor(self._schema) as accessor:
# This is the normal processing route
if self._manifest:
# Override the bill date to correspond with the manifest
bill_date = self._manifest.billing_period_start_datetime.date()
bills = accessor.get_cost_entry_bills_query_by_provider(
self._provider.uuid)
bills = bills.filter(billing_period_start=bill_date).all()
first_bill = bills.filter(
billing_period_start=bill_date).first()
do_month_update = False
with schema_context(self._schema):
if first_bill:
do_month_update = self._determine_if_full_summary_update_needed(
first_bill)
if do_month_update:
last_day_of_month = calendar.monthrange(
bill_date.year, bill_date.month)[1]
start_date = bill_date.strftime("%Y-%m-%d")
end_date = bill_date.replace(day=last_day_of_month)
end_date = end_date.strftime("%Y-%m-%d")
LOG.info(
"Overriding start and end date to process full month.")
return start_date, end_date
def update_summary_cost_model_costs(self, start_date=None, end_date=None):
"""Update the AWS summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
LOG.debug(
"Starting charge calculation updates for provider: %s. Dates: %s-%s",
self._provider.uuid,
str(start_date),
str(end_date),
)
self._update_markup_cost(start_date, end_date)
with AWSReportDBAccessor(self._schema) as accessor:
LOG.debug(
"Updating AWS derived cost summary for schema: %s and provider: %s",
self._schema, self._provider.uuid)
bills = accessor.bills_for_provider_uuid(self._provider.uuid,
start_date)
with schema_context(self._schema):
for bill in bills:
bill.derived_cost_datetime = DateAccessor(
).today_with_timezone("UTC")
bill.save()
def update_daily_tables(self, start_date, end_date):
"""Populate the daily tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str): A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema,
datetime.datetime.strptime(start_date, "%Y-%m-%d"),
datetime.datetime.strptime(end_date, "%Y-%m-%d"),
)
bill_ids = []
with schema_context(self._schema):
bill_ids = [str(bill.id) for bill in bills]
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating AWS report daily tables for \n\tSchema: %s"
"\n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
start,
end,
)
accessor.populate_line_item_daily_table(start, end, bill_ids)
return start_date, end_date
