def test_get_report_files_second_task_not_called(self, mock_process_files, mock_get_files): """Test that the chained task is not called.""" mock_process_files.delay = Mock() get_report_files(**self.fake_get_report_args) mock_process_files.delay.assert_not_called()
def test_process_report_files_with_transaction_atomic_error( self, mock_files, mock_processor): """Test than an exception rolls back the atomic transaction.""" path = "{}/{}".format("test", "file1.csv") mock_files.return_value = [{"file": path, "compression": "GZIP"}] schema_name = self.schema provider = Provider.PROVIDER_AWS provider_uuid = self.aws_provider_uuid report_month = DateHelper().today manifest_dict = { "assembly_id": "12345", "billing_period_start_datetime": report_month, "num_total_files": 1, "provider_uuid": self.aws_provider_uuid, "task": "170653c0-3e66-4b7e-a764-336496d7ca5a", } with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.add(**manifest_dict) manifest.save() manifest_id = manifest.id initial_update_time = manifest.manifest_updated_datetime with ReportStatsDBAccessor("file1.csv", manifest_id) as stats_accessor: stats_accessor.get_last_completed_datetime with ReportStatsDBAccessor(path, manifest_id) as report_file_accessor: report_file_accessor.get_last_started_datetime() mock_processor.side_effect = Exception with self.assertRaises(Exception): customer_name = "Fake Customer" authentication = "auth" billing_source = "bill" provider_type = provider get_report_files( customer_name=customer_name, authentication=authentication, billing_source=billing_source, provider_type=provider_type, schema_name=schema_name, provider_uuid=provider_uuid, report_month=report_month, ) with ReportStatsDBAccessor(path, manifest_id) as report_file_accessor: self.assertIsNone( report_file_accessor.get_last_completed_datetime()) with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(manifest_id) self.assertEqual(manifest.num_processed_files, 0) self.assertEqual(manifest.manifest_updated_datetime, initial_update_time) with ProviderDBAccessor( provider_uuid=provider_uuid) as provider_accessor: self.assertFalse(provider_accessor.get_setup_complete())
def test_get_report_exception(self, mock_process_files, mock_get_files, mock_started, mock_completed): """Test raising processor exception is handled.""" mock_get_files.return_value = self.fake_reports mock_started.return_value = None # Check that exception is raised with self.assertRaises(ReportProcessorError): # Check that the exception logs an ERROR with self.assertLogs("masu.processor.tasks.get_report_files", level="ERROR"): get_report_files(**self.fake_get_report_args)
def test_get_report_files_timestamps_empty_both(self, mock_process_files, mock_get_files, mock_started, mock_completed): """ Test that the chained task is called when no timestamps are set. """ mock_process_files.delay = Mock() mock_get_files.return_value = self.fake_reports mock_started.return_value = None mock_completed.return_value = None get_report_files(**self.fake_get_report_args) mock_process_files.delay.assert_called()
def process_report(report): """ Process line item report and kick off summarization celery task. Args: report (Dict) - keys: value file: String, cluster_id: String, payload_date: DateTime, manifest_path: String, uuid: String, manifest_path: String Returns: None """ cluster_id = report.get('cluster_id') provider_uuid = utils.get_provider_uuid_from_cluster_id(cluster_id) if provider_uuid: LOG.info('Found provider_uuid: %s for cluster_id: %s', str(provider_uuid), str(cluster_id)) account = get_account(provider_uuid) LOG.info('Processing report for account %s', account) reports_to_summarize = get_report_files(**account) LOG.info('Processing complete for account %s', account) async_id = summarize_reports.delay(reports_to_summarize) LOG.info('Summarization celery uuid: %s', str(async_id)) else: LOG.error('Could not find provider_uuid for cluster_id: %s', str(cluster_id))
def test_get_report_files_timestamps_aligned(self, mock_started, mock_completed): """Test to return reports only when they have not been processed.""" mock_started.return_value = self.yesterday mock_completed.return_value = self.today reports = get_report_files(**self.get_report_args) self.assertEqual(reports, [])
def test_get_report_files_timestamps_misaligned(self, mock_started, mock_completed): """Test to return reports with misaligned timestamps.""" mock_started.return_value = self.today mock_completed.return_value = self.yesterday reports = get_report_files(**self.get_report_args) self.assertEqual(reports, [])
def test_get_report_files_timestamps_empty_end(self, mock_process_files, mock_get_files, mock_started, mock_completed): """ Test that the chained task is not called when no end time is set since processing is in progress. """ mock_process_files.delay = Mock() mock_get_files.return_value = self.fake_reports mock_started.return_value = self.today mock_completed.return_value = None get_report_files(**self.fake_get_report_args) mock_process_files.delay.assert_not_called()
def test_get_report_files_second_task_called(self, mock_process_files, mock_get_files): """Test that the chained task is called.""" mock_process_files.delay = Mock() mock_get_files.return_value = self.fake_reports get_report_files(**self.fake_get_report_args) expected_calls = [ call(self.fake_get_report_args.get('schema_name'), report['file'], report['compression']) for report in self.fake_reports ] mock_process_files.delay.has_calls(expected_calls, any_order=True)
def test_get_report_files_timestamps_empty_start(self, mock_process_files, mock_started, mock_completed): """Test that the chained task is called when no start time is set.""" mock_process_files.apply_async = Mock() mock_started.return_value = None mock_completed.return_value = self.today reports = get_report_files(**self.get_report_args) self.assertIsNotNone(reports)
def test_get_report_files_timestamps_misaligned(self, mock_process_files, mock_get_files, mock_started, mock_completed): """ Test that the chained task is called when start timestamp is before end timestamp. """ mock_process_files.delay = Mock() mock_get_files.return_value = self.fake_reports mock_started.return_value = self.today mock_completed.return_value = self.yesterday get_report_files(**self.fake_get_report_args) mock_process_files.delay.assert_not_called()
def test_get_report_files_timestamps_empty_both(self, mock_process_file, mock_started, mock_completed): """Test that the chained task is called when no timestamps are set.""" mock_process_file.return_value = None mock_started.return_value = None mock_completed.return_value = None reports = get_report_files(**self.get_report_args) self.assertIsNotNone(reports)
def test_get_report_files_timestamps_empty_end(self, mock_date, mock_started, mock_completed): """Chained task is not called when no end time is set since processing is in progress.""" mock_started.return_value = self.today # Make sure today() is only an hour from get_last_started_datetime (within 2 hr timeout) mock_date.return_value = self.today + timedelta(hours=1) mock_completed.return_value = None reports = get_report_files(**self.get_report_args) self.assertEqual(reports, [])
def test_get_report_files_timestamps_empty_end_no_timeout(self, mock_date, mock_process_files, mock_get_files, mock_started, mock_completed): """ Test that the chained task is not called when no end time is set since processing is in progress but completion timeout has not been reached. """ mock_process_files.delay = Mock() mock_get_files.return_value = self.fake_reports mock_started.return_value = self.today mock_completed.return_value = None mock_date.return_value = self.today + timedelta(hours=1) get_report_files(**self.fake_get_report_args) mock_process_files.delay.assert_not_called()
def test_get_report_files_timestamps_empty_end_timeout(self, mock_date, mock_process_files, mock_get_files, mock_started, mock_completed): """ Test that the chained task is called when no end time is set since processing has exceeded the timeout. """ mock_process_files.delay = Mock() mock_get_files.return_value = self.fake_reports mock_started.return_value = self.today mock_completed.return_value = None mock_date.return_value = self.today + timedelta(hours=3) get_report_files(**self.fake_get_report_args) mock_process_files.delay.assert_called()
def test_get_report_files_timestamps_empty_end_timeout( self, mock_date, mock_process_files, mock_started, mock_completed): """Chained task is called when no end time is set since processing has exceeded the timeout.""" mock_process_files.apply_async = Mock() mock_started.return_value = self.today mock_completed.return_value = None mock_date.return_value = self.today + timedelta(hours=3) reports = get_report_files(**self.get_report_args) self.assertIsNotNone(reports)
def test_get_report_files_timestamps_empty_end_no_timeout( self, mock_date, mock_process_files, mock_started, mock_completed): """ Chained task is not called when no end time is set. Since processing is in progress but completion timeout has not been reached. """ mock_started.return_value = self.today mock_completed.return_value = None mock_date.return_value = self.today + timedelta(hours=1) reports = get_report_files(**self.get_report_args) self.assertIsNotNone(reports)
def process_report(report): """ Process line item report and kick off summarization celery task. Args: report (Dict) - keys: value file: String, cluster_id: String, date: DateTime, manifest_path: String, uuid: String, manifest_path: String Returns: None """ cluster_id = report.get("cluster_id") provider_uuid = utils.get_provider_uuid_from_cluster_id(cluster_id) if provider_uuid: LOG.info("Found provider_uuid: %s for cluster_id: %s", str(provider_uuid), str(cluster_id)) account = get_account(provider_uuid) if account: payload_date = report.get("date") report_month = payload_date.replace(day=1, second=1, microsecond=1).date() account["report_month"] = str(report_month) LOG.info("Processing %s report for account %s", payload_date.strftime("%B %Y"), account) reports_to_summarize = get_report_files(**account) LOG.info("Processing complete for account %s", account) async_id = summarize_reports.delay(reports_to_summarize) LOG.info("Summarization celery uuid: %s", str(async_id)) else: LOG.error("Could not find provider_uuid for cluster_id: %s", str(cluster_id))