def test_delete_line_items_not_first_file_in_manifest(self):
"""Test that data is not deleted once a file has been processed."""
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
report_file = manifest_helper.generate_one_test_file()
manifest_helper.mark_report_file_as_completed(report_file)
processor = AWSReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=self.manifest.id,
)
processor.process()
result = processor._delete_line_items(AWSReportDBAccessor)
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(
bill_id)
self.assertFalse(result)
self.assertNotEqual(line_item_query.count(), 0)
def test_clean_volume(self, mock_date, mock_config):
"""Test that the clean volume function is cleaning the appropriate files"""
# create a manifest
mock_date.return_value = ["2020-02-01"]
manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": "2020-02-01",
"num_total_files": 2,
"provider_uuid": self.aws_provider_uuid,
}
manifest_accessor = ReportManifestDBAccessor()
manifest = manifest_accessor.add(**manifest_dict)
# create two files on the temporary volume one with a matching prefix id
# as the assembly_id in the manifest above
with tempfile.TemporaryDirectory() as tmpdirname:
mock_config.PVC_DIR = tmpdirname
mock_config.VOLUME_FILE_RETENTION = 60 * 60 * 24
old_matching_file = os.path.join(tmpdirname, "%s.csv" % manifest.assembly_id)
new_no_match_file = os.path.join(tmpdirname, "newfile.csv")
old_no_match_file = os.path.join(tmpdirname, "oldfile.csv")
filepaths = [old_matching_file, new_no_match_file, old_no_match_file]
for path in filepaths:
open(path, "a").close()
self.assertEqual(os.path.exists(path), True)
# Update timestame for oldfile.csv
datehelper = DateHelper()
now = datehelper.now
old_datetime = now - timedelta(seconds=mock_config.VOLUME_FILE_RETENTION * 2)
oldtime = old_datetime.timestamp()
os.utime(old_matching_file, (oldtime, oldtime))
os.utime(old_no_match_file, (oldtime, oldtime))
# now run the clean volume task
tasks.clean_volume()
# make sure that the file with the matching id still exists and that
# the file with the other id is gone
self.assertEqual(os.path.exists(old_matching_file), True)
self.assertEqual(os.path.exists(new_no_match_file), True)
self.assertEqual(os.path.exists(old_no_match_file), False)
# now edit the manifest to say that all the files have been processed
# and rerun the clean_volumes task
manifest.num_processed_files = manifest_dict.get("num_total_files")
manifest_helper = ManifestCreationHelper(
manifest.id, manifest_dict.get("num_total_files"), manifest_dict.get("assembly_id")
)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
manifest.save()
tasks.clean_volume()
# ensure that the original file is deleted from the volume
self.assertEqual(os.path.exists(old_matching_file), False)
self.assertEqual(os.path.exists(new_no_match_file), True)
# assert the tempdir is cleaned up
self.assertEqual(os.path.exists(tmpdirname), False)
# test no files found for codecov
tasks.clean_volume()
def test_update_summary_tables_finalized_bill(self, mock_daily,
mock_summary):
"""Test that summary tables are run for a full month."""
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.finalized_datetime = start_date
bill.save()
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = start_date.replace(day=1)
expected_end_date = end_date.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
if expected_end_date not in dates:
dates.append(expected_end_date)
# Remove the first date since it's the start date
expected_start_date = dates.pop(0)
expected_calls = []
for date in dates:
if expected_start_date > expected_end_date:
break
expected_calls.append(
call(expected_start_date.date(), date.date(), [str(bill.id)]))
expected_start_date = date + datetime.timedelta(days=1)
self.updater.update_daily_tables(start_date_str, end_date_str)
self.assertEqual(mock_daily.call_args_list, expected_calls)
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_summary.call_args_list, expected_calls)
with AWSReportDBAccessor(self.schema) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)
def test_simulate_delete_expired_cost_usage_report_manifest_by_provider_uuid(
self):
"""
Test simulating the deletion of expired CostUsageReportManifests.
using remove(provider_uuid)
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
expiration_date = remover._calculate_expiration_date()
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
manifest_id = 7766
day_before_cutoff_data = {
"id": manifest_id,
"assembly_id": uuid4(),
"manifest_creation_datetime": None,
"manifest_updated_datetime": None,
"billing_period_start_datetime": day_before_cutoff,
"num_total_files": 1,
"provider_id": self.aws_provider_uuid,
}
manifest_entry = CostUsageReportManifest(**day_before_cutoff_data)
manifest_entry.save()
manifest_helper = ManifestCreationHelper(
manifest_id, manifest_entry.num_total_files,
manifest_entry.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
count_records = CostUsageReportManifest.objects.count()
with self.assertLogs(logger="masu.processor.expired_data_remover",
level="INFO") as cm:
logging.disable(logging.NOTSET)
remover.remove(simulate=True, provider_uuid=self.aws_provider_uuid)
expected_log_message = "Removed CostUsageReportManifest"
# Check if the log message exists in the log output:
self.assertTrue(
any(match is not None for match in [
re.search(expected_log_message, line) for line in cm.output
]),
"Expected to see log message: " + expected_log_message +
"in the list of log messages" +
" but the list of log messages was instead : " +
str(cm.output),
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
self.assertEqual(count_records,
CostUsageReportManifest.objects.count())
def test_get_last_seen_manifest_ids(self):
"""Test that get_last_seen_manifest_ids returns the appropriate assembly_ids."""
# test that the most recently seen manifests that haven't been processed are returned
manifest_dict2 = {
"assembly_id": "5678",
"billing_period_start_datetime": self.billing_start,
"num_total_files": 1,
"provider_uuid": "00000000-0000-0000-0000-000000000002",
}
manifest = self.manifest_accessor.add(**self.manifest_dict)
manifest2 = self.manifest_accessor.add(**manifest_dict2)
assembly_ids = self.manifest_accessor.get_last_seen_manifest_ids(
self.billing_start)
self.assertEqual(assembly_ids,
[manifest.assembly_id, manifest2.assembly_id])
# test that when the manifest's files have been processed - it is no longer returned
manifest2_helper = ManifestCreationHelper(
manifest2.id, manifest_dict2.get("num_total_files"),
manifest_dict2.get("assembly_id"))
manifest2_helper.generate_test_report_files()
manifest2_helper.process_all_files()
assembly_ids = self.manifest_accessor.get_last_seen_manifest_ids(
self.billing_start)
self.assertEqual(assembly_ids, [manifest.assembly_id])
# test that of two manifests with the same provider_ids - that only the most recently
# seen is returned
manifest_dict3 = {
"assembly_id": "91011",
"billing_period_start_datetime": self.billing_start,
"num_total_files": 1,
"provider_uuid": self.provider_uuid,
}
manifest3 = self.manifest_accessor.add(**manifest_dict3)
assembly_ids = self.manifest_accessor.get_last_seen_manifest_ids(
self.billing_start)
self.assertEqual(assembly_ids, [manifest3.assembly_id])
# test that manifests for a different billing month are not returned
current_month = self.billing_start
calculated_month = current_month + relativedelta(months=-2)
manifest3.billing_period_start_datetime = calculated_month
manifest3.save()
assembly_ids = self.manifest_accessor.get_last_seen_manifest_ids(
self.billing_start)
self.assertEqual(assembly_ids, [manifest.assembly_id])
def test_update_summary_tables_new_period_last_month(
self, mock_sum, mock_storage_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.dh.today.replace(day=1) + relativedelta(months=-1)
manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"provider_uuid": self.ocp_provider_uuid,
}
self.manifest_accessor.delete(self.manifest)
self.manifest = self.manifest_accessor.add(**manifest_dict)
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
self.manifest.save()
self.updater = OCPReportSummaryUpdater(self.schema, self.provider,
self.manifest)
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date).filter(
provider_id=self.ocp_provider_uuid,
report_period_start=billing_start)[0]
period.summary_data_creation_datetime = None
period.summary_data_updated_datetime = None
period.save()
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = billing_start
expected_end_date = billing_start.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
if expected_end_date not in dates:
dates.append(expected_end_date)
# Remove the first date since it's the start date
expected_start_date = dates.pop(0)
expected_calls = []
expected_calls_with_source_uuid = []
for date in dates:
if expected_start_date > expected_end_date:
break
expected_calls.append(
call(expected_start_date.date(), date.date(), self.cluster_id))
expected_calls_with_source_uuid.append(
call(expected_start_date.date(), date.date(), self.cluster_id,
self.provider.uuid))
expected_start_date = date + datetime.timedelta(days=1)
self.assertIsNone(period.summary_data_creation_datetime)
self.assertIsNone(period.summary_data_updated_datetime)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_sum.call_args_list,
expected_calls_with_source_uuid)
self.assertEqual(mock_storage_summary.call_args_list,
expected_calls_with_source_uuid)
def test_update_summary_tables_new_bill_last_month(self, mock_daily,
mock_summary):
"""Test that summary tables are run for the month of the manifest."""
billing_start = self.date_accessor.today_with_timezone("UTC").replace(
day=1) + relativedelta(months=-1)
manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"provider_uuid": self.aws_provider_uuid,
}
self.manifest_accessor.delete(self.manifest)
self.manifest = self.manifest_accessor.add(**manifest_dict)
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
self.updater = AWSReportSummaryUpdater(self.schema, self.aws_provider,
self.manifest)
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = billing_start.date()
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills_by_date(bill_date)
for bill in bills:
bill.summary_data_creation_datetime = None
bill.save()
bill_ids = sorted((str(bill.id) for bill in bills), reverse=True)
last_day_of_month = calendar.monthrange(bill_date.year,
bill_date.month)[1]
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = billing_start
expected_end_date = billing_start.replace(day=last_day_of_month)
dates = list(
rrule(freq=DAILY,
dtstart=expected_start_date,
until=expected_end_date,
interval=5))
if expected_end_date not in dates:
dates.append(expected_end_date)
# Remove the first date since it's the start date
expected_start_date = dates.pop(0)
expected_calls = []
for date in dates:
if expected_start_date > expected_end_date:
break
expected_calls.append(
call(expected_start_date.date(), date.date(), bill_ids))
expected_start_date = date + datetime.timedelta(days=1)
self.updater.update_daily_tables(start_date_str, end_date_str)
self.assertEqual(mock_daily.call_args_list, expected_calls)
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
self.assertEqual(mock_summary.call_args_list, expected_calls)
with AWSReportDBAccessor(self.schema) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)