def setUp(self):
"""Set up each test."""
super().setUp()
# with ProviderDBAccessor(self.ocp_test_provider_uuid) as provider_accessor:
# self.provider = provider_accessor.get_provider()
self.provider = self.ocp_provider
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
day=1)
self.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'num_processed_files': 1,
'provider_id': self.ocp_provider.id,
}
today = DateAccessor().today_with_timezone('UTC')
cluster_id = self.ocp_provider_resource_name
self.report_period = self.creator.create_ocp_report_period(
today, provider_id=self.provider.id, cluster_id=cluster_id)
report = self.creator.create_ocp_report(self.report_period, today)
self.creator.create_ocp_usage_line_item(self.report_period, report)
self.creator.create_ocp_storage_line_item(self.report_period, report)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
def setUp(self):
"""Set up each test."""
super().setUp()
if self.accessor._conn.closed:
self.accessor._conn = self.accessor._db.connect()
if self.accessor._pg2_conn.closed:
self.accessor._pg2_conn = self.accessor._get_psycopg2_connection()
if self.accessor._cursor.closed:
self.accessor._cursor = self.accessor._get_psycopg2_cursor()
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
self.provider = provider_accessor.get_provider()
today = DateAccessor().today_with_timezone('UTC')
cluster_id = self.ocp_provider_resource_name
self.report_period = self.creator.create_ocp_report_period(
today, provider_id=self.provider.id, cluster_id=cluster_id)
report = self.creator.create_ocp_report(self.report_period, today)
self.creator.create_ocp_usage_line_item(self.report_period, report)
self.creator.create_ocp_storage_line_item(self.report_period, report)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
self.updater = OCPReportSummaryUpdater('acct10001', self.provider,
self.manifest)
def setUp(self):
"""Set up each test."""
super().setUp()
self.provider = self.ocp_provider
self.today = self.dh.today
billing_start = datetime.datetime(year=self.today.year,
month=self.today.month,
day=self.today.day).replace(day=1)
self.manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"num_processed_files": 1,
"provider_uuid": self.ocp_provider_uuid,
}
self.cluster_id = self.ocp_cluster_id
self.manifest = CostUsageReportManifest.objects.filter(
provider_id=self.ocp_provider_uuid,
billing_period_start_datetime=self.dh.this_month_start).first()
self.manifest.num_total_files = 2
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
def setUp(self):
"""Set up each test."""
super().setUp()
self.provider = self.ocp_provider
self.today = DateAccessor().today_with_timezone("UTC")
billing_start = datetime.datetime(year=self.today.year,
month=self.today.month,
day=self.today.day).replace(day=1)
self.manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"num_processed_files": 1,
"provider_uuid": self.ocp_provider_uuid,
}
cluster_id = self.ocp_provider_resource_name
self.report_period = self.creator.create_ocp_report_period(
provider_uuid=self.ocp_provider_uuid,
period_date=self.today,
cluster_id=cluster_id)
report = self.creator.create_ocp_report(self.report_period, self.today)
self.creator.create_ocp_usage_line_item(self.report_period, report)
self.creator.create_ocp_storage_line_item(self.report_period, report)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
def _set_updater(self):
"""
Create the report summary updater object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report summary updater
"""
if self._provider.type in (AMAZON_WEB_SERVICES,
AWS_LOCAL_SERVICE_PROVIDER):
return (AWSReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (AZURE, AZURE_LOCAL_SERVICE_PROVIDER):
return (AzureReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (OPENSHIFT_CONTAINER_PLATFORM, ):
return (OCPReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
return None
def _set_updater(self):
"""
Create the report summary updater object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report summary updater
"""
if self._provider.type in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
return (AWSReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
return (AzureReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (Provider.PROVIDER_OCP, ):
return (OCPReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
return (None, None)
def test_update_summary_tables_without_manifest(
self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary
):
"""Test that summary tables are properly run without a manifest."""
# Create an updater that doesn't have a manifest
updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
start_date = DateAccessor().today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_updated_datetime = start_date
period.save()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
mock_storage_daily.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
mock_storage_summary.assert_called_with(
expected_start_date, expected_end_date, self.report_period.cluster_id
)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertGreater(period.summary_data_updated_datetime, start_date)
class OCPReportSummaryUpdaterTest(MasuTestCase):
"""Test cases for the OCPReportSummaryUpdater class."""
@classmethod
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor(cls.schema, cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
@classmethod
def tearDownClass(cls):
"""Tear down the test class."""
super().tearDownClass()
cls.accessor.close_connections()
def setUp(self):
"""Set up each test."""
super().setUp()
# with ProviderDBAccessor(self.ocp_test_provider_uuid) as provider_accessor:
# self.provider = provider_accessor.get_provider()
self.provider = self.ocp_provider
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
day=1)
self.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'num_processed_files': 1,
'provider_id': self.ocp_provider.id,
}
today = DateAccessor().today_with_timezone('UTC')
cluster_id = self.ocp_provider_resource_name
self.report_period = self.creator.create_ocp_report_period(
today, provider_id=self.provider.id, cluster_id=cluster_id)
report = self.creator.create_ocp_report(self.report_period, today)
self.creator.create_ocp_usage_line_item(self.report_period, report)
self.creator.create_ocp_storage_line_item(self.report_period, report)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
def tearDown(self):
"""Return the database to a pre-test state."""
super().tearDown()
# def run(self, result=None):
# """Run the tests with the correct schema context."""
# with schema_context(self.schema):
# super().run(result)
# for table_name in self.all_tables:
# tables = self.accessor._get_db_obj_query(table_name).all()
# for table in tables:
# self.accessor.delete(table)
# self.accessor.commit()
# manifests = self.manifest_accessor._get_db_obj_query().all()
# for manifest in manifests:
# self.manifest_accessor.delete(manifest)
# self.manifest_accessor.commit()
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table'
)
def test_update_summary_tables_with_manifest(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are properly run."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
# self.manifest_accessor.commit()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_creation_datetime = start_date
period.save()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table'
)
def test_update_summary_tables_new_period(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are run for a full month."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
# self.manifest_accessor.commit()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.replace(day=1).strftime('%Y-%m-%d')
expected_end_date = end_date.replace(
day=last_day_of_month).strftime('%Y-%m-%d')
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table'
)
def test_update_summary_tables_new_period_last_month(
self, mock_daily, mock_sum, mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
day=1) + relativedelta(months=-1)
manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': self.ocp_provider.id,
}
self.manifest_accessor.delete(self.manifest)
# self.manifest_accessor.commit()
self.manifest = self.manifest_accessor.add(**manifest_dict)
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
self.creator.create_ocp_report_period(billing_start,
provider_id=self.provider.id)
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = bill_date.strftime('%Y-%m-%d')
expected_end_date = bill_date.replace(
day=last_day_of_month).strftime('%Y-%m-%d')
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table'
)
def test_update_summary_tables_existing_period_done_processing(
self, mock_daily, mock_sum, mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are not run for a full month."""
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_creation_datetime = start_date
period.save()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table'
)
def test_update_summary_tables_without_manifest(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are properly run without a manifest."""
# Create an updater that doesn't have a manifest
updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
start_date = DateAccessor().today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_updated_datetime = start_date
period.save()
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.strftime('%Y-%m-%d')
expected_end_date = end_date.strftime('%Y-%m-%d')
updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertGreater(period.summary_data_updated_datetime,
start_date)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.get_usage_period_query_by_provider'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table'
)
@patch(
'masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table'
)
def test_update_summary_tables_no_period(
self,
mock_daily,
mock_sum,
mock_period,
mock_storage_daily,
mock_storage_summary,
):
"""Test that summary tables are run for a full month when no report period is found."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date
mock_period_filter_by = Mock()
mock_period_filter_by.all.return_value = None
mock_period.filter_by.return_value = mock_period_filter_by
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called()
mock_storage_daily.assert_called()
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called()
mock_storage_summary.assert_called()
def test_update_summary_tables_new_period_last_month(
self, mock_daily, mock_sum, mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
day=1) + relativedelta(months=-1)
manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': self.ocp_provider.id,
}
self.manifest_accessor.delete(self.manifest)
# self.manifest_accessor.commit()
self.manifest = self.manifest_accessor.add(**manifest_dict)
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
self.creator.create_ocp_report_period(billing_start,
provider_id=self.provider.id)
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = bill_date.strftime('%Y-%m-%d')
expected_end_date = bill_date.replace(
day=last_day_of_month).strftime('%Y-%m-%d')
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(expected_start_date, expected_end_date,
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(expected_start_date,
expected_end_date,
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
class OCPReportSummaryUpdaterTest(MasuTestCase):
"""Test cases for the OCPReportSummaryUpdater class."""
@classmethod
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.accessor = OCPReportDBAccessor(cls.schema)
cls.report_schema = cls.accessor.report_schema
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.schema)
cls.date_accessor = DateHelper()
cls.manifest_accessor = ReportManifestDBAccessor()
cls.dh = DateHelper()
def setUp(self):
"""Set up each test."""
super().setUp()
self.provider = self.ocp_provider
self.today = self.dh.today
billing_start = datetime.datetime(year=self.today.year,
month=self.today.month,
day=self.today.day).replace(day=1)
self.manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"num_processed_files": 1,
"provider_uuid": self.ocp_provider_uuid,
}
self.cluster_id = self.ocp_cluster_id
self.manifest = CostUsageReportManifest.objects.filter(
provider_id=self.ocp_provider_uuid,
billing_period_start_datetime=self.dh.this_month_start).first()
self.manifest.num_total_files = 2
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_storage_line_item_daily_summary_table"
)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_line_item_daily_summary_table")
def test_update_summary_tables_with_manifest(self, mock_sum,
mock_storage_summary):
"""Test that summary tables are properly run."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_creation_datetime = start_date
period.summary_data_updated_datetime = None
period.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
self.assertIsNone(period.summary_data_updated_datetime)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called()
mock_storage_summary.assert_called()
with OCPReportDBAccessor(self.schema) as accessor:
period = accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_node_label_line_item_daily_table"
)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_storage_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table"
)
def test_update_summary_tables_new_period(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary,
mock_node_daily):
"""Test that summary tables are run for a full month."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
start_date = self.dh.today
end_date = start_date
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid)[0]
period.summary_data_creation_datetime = None
period.summary_data_updated_datetime = None
period.save()
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = start_date.replace(day=1)
expected_end_date = end_date.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
if expected_end_date not in dates:
dates.append(expected_end_date)
# Remove the first date since it's the start date
dates.pop(0)
expected_calls = []
expected_calls_with_source_uuid = []
for date in dates:
if expected_start_date > expected_end_date:
break
expected_calls.append(
call(expected_start_date.date(), date.date(), self.cluster_id))
expected_calls_with_source_uuid.append(
call(expected_start_date.date(), date.date(), self.cluster_id,
self.provider.uuid))
expected_start_date = date + datetime.timedelta(days=1)
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
self.assertEqual(mock_node_daily.call_args_list, expected_calls)
self.assertEqual(mock_daily.call_args_list, expected_calls)
self.assertEqual(mock_storage_daily.call_args_list, expected_calls)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_sum.call_args_list,
expected_calls_with_source_uuid)
self.assertEqual(mock_storage_summary.call_args_list,
expected_calls_with_source_uuid)
with OCPReportDBAccessor(self.schema) as accessor:
period = accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_storage_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table"
)
def test_update_summary_tables_new_period_last_month(
self, mock_sum, mock_storage_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.dh.today.replace(day=1) + relativedelta(months=-1)
manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"provider_uuid": self.ocp_provider_uuid,
}
self.manifest_accessor.delete(self.manifest)
self.manifest = self.manifest_accessor.add(**manifest_dict)
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid,
report_period_start=billing_start)[0]
period.summary_data_creation_datetime = None
period.summary_data_updated_datetime = None
period.save()
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = billing_start
expected_end_date = billing_start.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
if expected_end_date not in dates:
dates.append(expected_end_date)
# Remove the first date since it's the start date
expected_start_date = dates.pop(0)
expected_calls = []
expected_calls_with_source_uuid = []
for date in dates:
if expected_start_date > expected_end_date:
break
expected_calls.append(
call(expected_start_date.date(), date.date(), self.cluster_id))
expected_calls_with_source_uuid.append(
call(expected_start_date.date(), date.date(), self.cluster_id,
self.provider.uuid))
expected_start_date = date + datetime.timedelta(days=1)
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_sum.call_args_list,
expected_calls_with_source_uuid)
self.assertEqual(mock_storage_summary.call_args_list,
expected_calls_with_source_uuid)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_storage_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table"
)
def test_update_summary_tables_existing_period_done_processing(
self, mock_sum, mock_storage_summary):
"""Test that summary tables are not run for a full month."""
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_creation_datetime = start_date
period.summary_data_updated_datetime = None
period.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
self.assertIsNone(period.summary_data_updated_datetime)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called()
mock_storage_summary.assert_called()
with OCPReportDBAccessor(self.schema) as accessor:
period = accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_node_label_line_item_daily_table"
)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_storage_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table"
)
def test_update_summary_tables_without_manifest(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary,
mock_node_daily):
"""Test that summary tables are properly run without a manifest."""
# Create an updater that doesn't have a manifest
updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
start_date = DateAccessor().today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_updated_datetime = start_date
period.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
updater.update_daily_tables(start_date_str, end_date_str)
mock_node_daily.assert_called_with(start_date.date(), end_date.date(),
self.cluster_id)
mock_daily.assert_called_with(start_date.date(), end_date.date(),
self.cluster_id)
mock_storage_daily.assert_called_with(start_date.date(),
end_date.date(), self.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(start_date.date(), end_date.date(),
self.cluster_id, self.provider.uuid)
mock_storage_summary.assert_called_with(start_date.date(),
end_date.date(),
self.cluster_id,
self.provider.uuid)
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertGreater(period.summary_data_updated_datetime, self.today)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_volume_label_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_pod_label_summary_table"
)
@patch("masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor."
"update_line_item_daily_summary_with_enabled_tags")
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_node_label_line_item_daily_table"
)
@patch("masu.processor.ocp.ocp_report_summary_updater."
"OCPReportDBAccessor.populate_storage_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_storage_line_item_daily_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.get_usage_period_query_by_provider"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_summary_table"
)
@patch(
"masu.processor.ocp.ocp_report_summary_updater.OCPReportDBAccessor.populate_line_item_daily_table"
)
def test_update_summary_tables_no_period(
self,
mock_daily,
mock_sum,
mock_period,
mock_storage_daily,
mock_storage_summary,
mock_node_daily,
mock_udt_lids_tags,
mock_pod_labels,
mock_volume_labels,
):
"""Test that summary tables are run for a full month when no report period is found."""
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
start_date = self.dh.today
end_date = start_date
mock_period_filter_by = Mock()
mock_period_filter_by.all.return_value = None
mock_period.filter_by.return_value = mock_period_filter_by
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_node_daily.assert_called()
mock_daily.assert_called()
mock_storage_daily.assert_called()
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
mock_udt_lids_tags.not_assert_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called()
mock_storage_summary.assert_called()
mock_udt_lids_tags.assert_called()
mock_pod_labels.assert_called()
mock_volume_labels.assert_called()
def test_update_summary_tables_new_period_last_month(
self, mock_sum, mock_storage_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.dh.today.replace(day=1) + relativedelta(months=-1)
manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"provider_uuid": self.ocp_provider_uuid,
}
self.manifest_accessor.delete(self.manifest)
self.manifest = self.manifest_accessor.add(**manifest_dict)
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid,
report_period_start=billing_start)[0]
period.summary_data_creation_datetime = None
period.summary_data_updated_datetime = None
period.save()
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = billing_start
expected_end_date = billing_start.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
if expected_end_date not in dates:
dates.append(expected_end_date)
# Remove the first date since it's the start date
expected_start_date = dates.pop(0)
expected_calls = []
expected_calls_with_source_uuid = []
for date in dates:
if expected_start_date > expected_end_date:
break
expected_calls.append(
call(expected_start_date.date(), date.date(), self.cluster_id))
expected_calls_with_source_uuid.append(
call(expected_start_date.date(), date.date(), self.cluster_id,
self.provider.uuid))
expected_start_date = date + datetime.timedelta(days=1)
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_sum.call_args_list,
expected_calls_with_source_uuid)
self.assertEqual(mock_storage_summary.call_args_list,
expected_calls_with_source_uuid)
def test_update_summary_tables_new_period_last_month(
self, mock_daily, mock_sum, mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.date_accessor.today_with_timezone("UTC").replace(
day=1) + relativedelta(months=-1)
manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"provider_uuid": self.ocp_provider_uuid,
}
self.manifest_accessor.delete(self.manifest)
self.manifest = self.manifest_accessor.add(**manifest_dict)
self.manifest.num_processed_files = self.manifest.num_total_files
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
self.creator.create_ocp_report_period(
provider_uuid=self.ocp_provider_uuid, period_date=billing_start)
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = bill_date
expected_end_date = bill_date.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
# Remove the first date since it's the start date
expected_start_date = dates.pop(0)
expected_calls = []
for date in dates:
expected_calls.append(
call(expected_start_date.date(), date.date(),
self.report_period.cluster_id))
expected_start_date = date + datetime.timedelta(days=1)
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
self.assertEqual(mock_daily.call_args_list, expected_calls)
self.assertEqual(mock_storage_daily.call_args_list, expected_calls)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_sum.call_args_list, expected_calls)
self.assertEqual(mock_storage_summary.call_args_list, expected_calls)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertIsNotNone(period.summary_data_updated_datetime)