def test_get_report_files_second_task_not_called(self,
mock_process_files,
mock_get_files):
"""Test that the chained task is not called."""
mock_process_files.delay = Mock()
get_report_files(**self.fake_get_report_args)
mock_process_files.delay.assert_not_called()
def test_process_report_files_with_transaction_atomic_error(
self, mock_files, mock_processor):
"""Test than an exception rolls back the atomic transaction."""
path = "{}/{}".format("test", "file1.csv")
mock_files.return_value = [{"file": path, "compression": "GZIP"}]
schema_name = self.schema
provider = Provider.PROVIDER_AWS
provider_uuid = self.aws_provider_uuid
report_month = DateHelper().today
manifest_dict = {
"assembly_id": "12345",
"billing_period_start_datetime": report_month,
"num_total_files": 1,
"provider_uuid": self.aws_provider_uuid,
"task": "170653c0-3e66-4b7e-a764-336496d7ca5a",
}
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.add(**manifest_dict)
manifest.save()
manifest_id = manifest.id
initial_update_time = manifest.manifest_updated_datetime
with ReportStatsDBAccessor("file1.csv", manifest_id) as stats_accessor:
stats_accessor.get_last_completed_datetime
with ReportStatsDBAccessor(path, manifest_id) as report_file_accessor:
report_file_accessor.get_last_started_datetime()
mock_processor.side_effect = Exception
with self.assertRaises(Exception):
customer_name = "Fake Customer"
authentication = "auth"
billing_source = "bill"
provider_type = provider
get_report_files(
customer_name=customer_name,
authentication=authentication,
billing_source=billing_source,
provider_type=provider_type,
schema_name=schema_name,
provider_uuid=provider_uuid,
report_month=report_month,
)
with ReportStatsDBAccessor(path, manifest_id) as report_file_accessor:
self.assertIsNone(
report_file_accessor.get_last_completed_datetime())
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest_by_id(manifest_id)
self.assertEqual(manifest.num_processed_files, 0)
self.assertEqual(manifest.manifest_updated_datetime,
initial_update_time)
with ProviderDBAccessor(
provider_uuid=provider_uuid) as provider_accessor:
self.assertFalse(provider_accessor.get_setup_complete())
def test_get_report_exception(self, mock_process_files, mock_get_files, mock_started, mock_completed):
"""Test raising processor exception is handled."""
mock_get_files.return_value = self.fake_reports
mock_started.return_value = None
# Check that exception is raised
with self.assertRaises(ReportProcessorError):
# Check that the exception logs an ERROR
with self.assertLogs("masu.processor.tasks.get_report_files", level="ERROR"):
get_report_files(**self.fake_get_report_args)
def test_get_report_files_timestamps_empty_both(self,
mock_process_files,
mock_get_files,
mock_started,
mock_completed):
"""
Test that the chained task is called when no timestamps are set.
"""
mock_process_files.delay = Mock()
mock_get_files.return_value = self.fake_reports
mock_started.return_value = None
mock_completed.return_value = None
get_report_files(**self.fake_get_report_args)
mock_process_files.delay.assert_called()
def process_report(report):
"""
Process line item report and kick off summarization celery task.
Args:
report (Dict) - keys: value
file: String,
cluster_id: String,
payload_date: DateTime,
manifest_path: String,
uuid: String,
manifest_path: String
Returns:
None
"""
cluster_id = report.get('cluster_id')
provider_uuid = utils.get_provider_uuid_from_cluster_id(cluster_id)
if provider_uuid:
LOG.info('Found provider_uuid: %s for cluster_id: %s', str(provider_uuid), str(cluster_id))
account = get_account(provider_uuid)
LOG.info('Processing report for account %s', account)
reports_to_summarize = get_report_files(**account)
LOG.info('Processing complete for account %s', account)
async_id = summarize_reports.delay(reports_to_summarize)
LOG.info('Summarization celery uuid: %s', str(async_id))
else:
LOG.error('Could not find provider_uuid for cluster_id: %s', str(cluster_id))
def test_get_report_files_timestamps_aligned(self, mock_started, mock_completed):
"""Test to return reports only when they have not been processed."""
mock_started.return_value = self.yesterday
mock_completed.return_value = self.today
reports = get_report_files(**self.get_report_args)
self.assertEqual(reports, [])
def test_get_report_files_timestamps_misaligned(self, mock_started, mock_completed):
"""Test to return reports with misaligned timestamps."""
mock_started.return_value = self.today
mock_completed.return_value = self.yesterday
reports = get_report_files(**self.get_report_args)
self.assertEqual(reports, [])
def test_get_report_files_timestamps_empty_end(self,
mock_process_files,
mock_get_files,
mock_started,
mock_completed):
"""
Test that the chained task is not called when no end time is set since
processing is in progress.
"""
mock_process_files.delay = Mock()
mock_get_files.return_value = self.fake_reports
mock_started.return_value = self.today
mock_completed.return_value = None
get_report_files(**self.fake_get_report_args)
mock_process_files.delay.assert_not_called()
def test_get_report_files_second_task_called(self,
mock_process_files,
mock_get_files):
"""Test that the chained task is called."""
mock_process_files.delay = Mock()
mock_get_files.return_value = self.fake_reports
get_report_files(**self.fake_get_report_args)
expected_calls = [
call(self.fake_get_report_args.get('schema_name'),
report['file'],
report['compression'])
for report in self.fake_reports
]
mock_process_files.delay.has_calls(expected_calls, any_order=True)
def test_get_report_files_timestamps_empty_start(self, mock_process_files, mock_started, mock_completed):
"""Test that the chained task is called when no start time is set."""
mock_process_files.apply_async = Mock()
mock_started.return_value = None
mock_completed.return_value = self.today
reports = get_report_files(**self.get_report_args)
self.assertIsNotNone(reports)
def test_get_report_files_timestamps_misaligned(self,
mock_process_files,
mock_get_files,
mock_started,
mock_completed):
"""
Test that the chained task is called when start timestamp is before
end timestamp.
"""
mock_process_files.delay = Mock()
mock_get_files.return_value = self.fake_reports
mock_started.return_value = self.today
mock_completed.return_value = self.yesterday
get_report_files(**self.fake_get_report_args)
mock_process_files.delay.assert_not_called()
def test_get_report_files_timestamps_empty_both(self, mock_process_file, mock_started, mock_completed):
"""Test that the chained task is called when no timestamps are set."""
mock_process_file.return_value = None
mock_started.return_value = None
mock_completed.return_value = None
reports = get_report_files(**self.get_report_args)
self.assertIsNotNone(reports)
def test_get_report_files_timestamps_empty_end(self, mock_date, mock_started, mock_completed):
"""Chained task is not called when no end time is set since processing is in progress."""
mock_started.return_value = self.today
# Make sure today() is only an hour from get_last_started_datetime (within 2 hr timeout)
mock_date.return_value = self.today + timedelta(hours=1)
mock_completed.return_value = None
reports = get_report_files(**self.get_report_args)
self.assertEqual(reports, [])
def test_get_report_files_timestamps_empty_end_no_timeout(self,
mock_date,
mock_process_files,
mock_get_files,
mock_started,
mock_completed):
"""
Test that the chained task is not called when no end time is set since
processing is in progress but completion timeout has not been reached.
"""
mock_process_files.delay = Mock()
mock_get_files.return_value = self.fake_reports
mock_started.return_value = self.today
mock_completed.return_value = None
mock_date.return_value = self.today + timedelta(hours=1)
get_report_files(**self.fake_get_report_args)
mock_process_files.delay.assert_not_called()
def test_get_report_files_timestamps_empty_end_timeout(self,
mock_date,
mock_process_files,
mock_get_files,
mock_started,
mock_completed):
"""
Test that the chained task is called when no end time is set since
processing has exceeded the timeout.
"""
mock_process_files.delay = Mock()
mock_get_files.return_value = self.fake_reports
mock_started.return_value = self.today
mock_completed.return_value = None
mock_date.return_value = self.today + timedelta(hours=3)
get_report_files(**self.fake_get_report_args)
mock_process_files.delay.assert_called()
def test_get_report_files_timestamps_empty_end_timeout(
self, mock_date, mock_process_files, mock_started, mock_completed):
"""Chained task is called when no end time is set since processing has exceeded the timeout."""
mock_process_files.apply_async = Mock()
mock_started.return_value = self.today
mock_completed.return_value = None
mock_date.return_value = self.today + timedelta(hours=3)
reports = get_report_files(**self.get_report_args)
self.assertIsNotNone(reports)
def test_get_report_files_timestamps_empty_end_no_timeout(
self, mock_date, mock_process_files, mock_started, mock_completed):
"""
Chained task is not called when no end time is set.
Since processing is in progress but completion timeout has not been reached.
"""
mock_started.return_value = self.today
mock_completed.return_value = None
mock_date.return_value = self.today + timedelta(hours=1)
reports = get_report_files(**self.get_report_args)
self.assertIsNotNone(reports)
def process_report(report):
"""
Process line item report and kick off summarization celery task.
Args:
report (Dict) - keys: value
file: String,
cluster_id: String,
date: DateTime,
manifest_path: String,
uuid: String,
manifest_path: String
Returns:
None
"""
cluster_id = report.get("cluster_id")
provider_uuid = utils.get_provider_uuid_from_cluster_id(cluster_id)
if provider_uuid:
LOG.info("Found provider_uuid: %s for cluster_id: %s",
str(provider_uuid), str(cluster_id))
account = get_account(provider_uuid)
if account:
payload_date = report.get("date")
report_month = payload_date.replace(day=1, second=1,
microsecond=1).date()
account["report_month"] = str(report_month)
LOG.info("Processing %s report for account %s",
payload_date.strftime("%B %Y"), account)
reports_to_summarize = get_report_files(**account)
LOG.info("Processing complete for account %s", account)
async_id = summarize_reports.delay(reports_to_summarize)
LOG.info("Summarization celery uuid: %s", str(async_id))
else:
LOG.error("Could not find provider_uuid for cluster_id: %s",
str(cluster_id))
