def test_ocp_on_cloud_processor(self, mock_trino_enabled): """Test that we return the right class.""" mock_trino_enabled.return_value = True processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, context={"request_id": 1}, ) self.assertIsInstance(processor.ocp_on_cloud_processor, OCPCloudParquetReportProcessor) mock_trino_enabled.reset_mock() mock_trino_enabled.return_value = False processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, context={"request_id": 1}, ) self.assertIsNone(processor.ocp_on_cloud_processor)
def _process_report_file(schema_name, provider, provider_uuid, report_dict): """ Task to process a Report. Args: schema_name (String) db schema name provider (String) provider type provider_uuid (String) provider uuid report_dict (dict) The report data dict from previous task Returns: None """ start_date = report_dict.get('start_date') report_path = report_dict.get('file') compression = report_dict.get('compression') manifest_id = report_dict.get('manifest_id') provider_id = report_dict.get('provider_id') stmt = ('Processing Report:' ' schema_name: {},' ' report_path: {},' ' compression: {},' ' provider: {},' ' start_date: {}') log_statement = stmt.format(schema_name, report_path, compression, provider, start_date) LOG.info(log_statement) mem = psutil.virtual_memory() mem_msg = 'Avaiable memory: {} bytes ({}%)'.format(mem.free, mem.percent) LOG.info(mem_msg) file_name = report_path.split('/')[-1] with ReportStatsDBAccessor(file_name, manifest_id) as stats_recorder: stats_recorder.log_last_started_datetime() processor = ReportProcessor(schema_name=schema_name, report_path=report_path, compression=compression, provider=provider, provider_id=provider_id, manifest_id=manifest_id) processor.process() with ReportStatsDBAccessor(file_name, manifest_id) as stats_recorder: stats_recorder.log_last_completed_datetime() with ReportManifestDBAccessor() as manifest_accesor: manifest = manifest_accesor.get_manifest_by_id(manifest_id) if manifest: manifest.num_processed_files += 1 manifest.save() manifest_accesor.mark_manifest_as_updated(manifest) else: LOG.error('Unable to find manifest for ID: %s, file %s', manifest_id, file_name) with ProviderDBAccessor(provider_uuid=provider_uuid) as provider_accessor: provider_accessor.setup_complete() files = processor.remove_processed_files(path.dirname(report_path)) LOG.info('Temporary files removed: %s', str(files))
def test_process_gzip(self): """Test the processing of a gzip compressed file.""" counts = {} processor = ReportProcessor(schema_name='testcustomer', report_path=self.test_report_gzip, compression=GZIP_COMPRESSED) report_db = processor.report_db report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) count = report_db._session.query(table).count() counts[table_name] = count processor.process() for table_name in self.report_tables: table = getattr(report_schema, table_name) count = report_db._session.query(table).count() if table_name == 'reporting_awscostentryreservation': self.assertTrue(count >= counts[table_name]) else: self.assertTrue(count > counts[table_name]) self.assertTrue(processor.report_db._conn.closed) self.assertTrue(processor.report_db._pg2_conn.closed)
def test_aws_remove_processed_files_error(self, fake_process): """Test to remove_processed_files for AWS with processing error""" processor = ReportProcessor(schema_name='acct10001', report_path='/my/report/file', compression='GZIP', provider=AMAZON_WEB_SERVICES, provider_id=1, manifest_id=None) with self.assertRaises(ReportProcessorError): processor.remove_processed_files('/my/report/file')
def test_aws_process_error(self, fake_process): """Test to process for AWS with processing error.""" processor = ReportProcessor( schema_name=self.schema, report_path='/my/report/file', compression='GZIP', provider=AMAZON_WEB_SERVICES, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) with self.assertRaises(ReportProcessorError): processor.process()
def test_aws_remove_processed_files_error(self, fake_process): """Test to remove_processed_files for AWS with processing error.""" processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) with self.assertRaises(ReportProcessorError): processor.remove_processed_files("/my/report/file")
def test_aws_remove_processed_files(self, fake_process): """Test to remove_processed_files for AWS""" processor = ReportProcessor(schema_name='acct10001', report_path='/my/report/file', compression='GZIP', provider=AMAZON_WEB_SERVICES, provider_id=1, manifest_id=None) try: processor.remove_processed_files('/my/report/file') except Exception: self.fail('unexpected error')
def test_aws_process(self, fake_process): """Test to process for AWS.""" processor = ReportProcessor( schema_name=self.schema, report_path='/my/report/file', compression='GZIP', provider=AMAZON_WEB_SERVICES, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) try: processor.process() except Exception: self.fail('unexpected error')
def test_aws_remove_processed_files(self, fake_process): """Test to remove_processed_files for AWS.""" processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) try: processor.remove_processed_files("/my/report/file") except Exception: self.fail("unexpected error")
def test_process_duplicates(self): """Test that row duplicates are not inserted into the DB.""" counts = {} processor = ReportProcessor(schema_name='testcustomer', report_path=self.test_report, compression=UNCOMPRESSED) # Process for the first time processor.process() report_db = processor.report_db report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) count = report_db._session.query(table).count() counts[table_name] = count processor = ReportProcessor(schema_name='testcustomer', report_path=self.test_report, compression=UNCOMPRESSED) # Process for the second time processor.process() for table_name in self.report_tables: table = getattr(report_schema, table_name) count = report_db._session.query(table).count() self.assertTrue(count == counts[table_name])
def test_aws_process(self, mock_process, mock_ocp_cloud_process, mock_parquet_process, mock_trino_enabled): """Test to process for AWS.""" mock_trino_enabled.return_value = True processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) processor.process() mock_process.assert_not_called() mock_parquet_process.assert_called() mock_ocp_cloud_process.assert_called() mock_trino_enabled.reset_mock() mock_parquet_process.reset_mock() mock_ocp_cloud_process.reset_mock() mock_trino_enabled.return_value = False processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) processor.process() mock_process.assert_called() mock_parquet_process.assert_not_called() mock_ocp_cloud_process.assert_not_called()
def process_report(self, report, compression, provider_type, provider, manifest): """Run the report processor on a report.""" status = baker.make("CostUsageReportStatus", manifest=manifest, report_name=report) status.last_started_datetime = self.dh.now ReportProcessor(self.schema, report, compression, provider_type, provider.uuid, manifest.id).process() status.last_completed_datetime = self.dh.now status.save()
def test_initializer_aws_local(self): """Test to initializer for AWS-local""" processor = ReportProcessor(schema_name='acct10001', report_path='/my/report/file', compression='GZIP', provider=AWS_LOCAL_SERVICE_PROVIDER, provider_id=1, manifest_id=None) self.assertIsNotNone(processor._processor)
def test_initializer_aws(self): """Test to initializer for AWS""" processor = ReportProcessor(schema_name='acct10001', report_path='/my/report/file', compression='GZIP', provider=AMAZON_WEB_SERVICES, provider_id=1, manifest_id=None) self.assertIsNotNone(processor._processor)
def test_initializer_invalid_provider(self): """Test to initializer with invalid provider""" with self.assertRaises(ReportProcessorError): ReportProcessor(schema_name='acct10001', report_path='/my/report/file', compression='GZIP', provider='unknown', provider_id=1, manifest_id=None)
def test_initializer_error(self, fake_processor): """Test to initializer with error.""" with self.assertRaises(ReportProcessorError): ReportProcessor(schema_name='acct10001', report_path='/my/report/file', compression='GZIP', provider=AMAZON_WEB_SERVICES, provider_id=1, manifest_id=None)
def test_initializer_invalid_provider(self): """Test to initializer with invalid provider.""" with self.assertRaises(ReportProcessorError): ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider="unknown", provider_uuid=self.aws_provider_uuid, manifest_id=None, )
def test_initializer_not_implemented_error(self, fake_processor): """Test to initializer with error.""" with self.assertRaises(NotImplementedError): ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, )
def test_initializer_error(self, fake_processor): """Test to initializer with error.""" with self.assertRaises(ReportProcessorError): ReportProcessor( schema_name=self.schema, report_path='/my/report/file', compression='GZIP', provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, )
def test_initializer_ocp_local(self): """Test to initializer for OCP-local""" processor = ReportProcessor( schema_name='acct10001', report_path= './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv', compression='PLAIN', provider=OCP_LOCAL_SERVICE_PROVIDER, provider_id=1, manifest_id=None) self.assertIsNotNone(processor._processor)
def test_initializer_ocp(self): """Test to initializer for OCP""" processor = ReportProcessor( schema_name='acct10001', report_path= './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv', compression='PLAIN', provider=OPENSHIFT_CONTAINER_PLATFORM, provider_id=1, manifest_id=None) self.assertIsNotNone(processor._processor)
def test_initializer_aws(self): """Test to initializer for AWS.""" processor = ReportProcessor( schema_name=self.schema, report_path='/my/report/file', compression='GZIP', provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) self.assertIsNotNone(processor._processor)
def test_initializer_aws_local(self): """Test to initializer for AWS-local.""" processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) self.assertIsNotNone(processor._processor)
def test_initializer_azure(self): """Test to initializer for Azure.""" processor = ReportProcessor( schema_name=self.schema, report_path='/my/report/file', compression='GZIP', provider=AZURE, provider_uuid=self.azure_provider_uuid, manifest_id=None, ) self.assertIsNotNone(processor._processor)
def test_initializer_azure_local(self): """Test to initializer for AZURE-local""" processor = ReportProcessor( schema_name=self.schema, report_path= './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv', compression='PLAIN', provider=AZURE_LOCAL_SERVICE_PROVIDER, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) self.assertIsNotNone(processor._processor)
def test_set_processor_parquet(self): """Test that the Parquet class is returned.""" processor = ReportProcessor( schema_name=self.schema, report_path="/my/report/file", compression="GZIP", provider=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, manifest_id=None, context={"request_id": 1}, ) self.assertIsInstance(processor._processor, ParquetReportProcessor)
def _process_report_file(schema_name, report_path, compression): """ Task to process a Report. Args: schema_name (String) db schema name report_path (String) path to downloaded reports compression (String) 'PLAIN' or 'GZIP' Returns: None """ stmt = ('Processing Report:' ' schema_name: {},' ' report_path: {},' ' compression: {}') log_statement = stmt.format(schema_name, report_path, compression) LOG.info(log_statement) mem = psutil.virtual_memory() mem_msg = 'Avaiable memory: {} bytes ({}%)'.format(mem.free, mem.percent) LOG.info(mem_msg) file_name = report_path.split('/')[-1] stats_recorder = ReportStatsDBAccessor(file_name) stats_recorder.log_last_started_datetime() stats_recorder.commit() processor = ReportProcessor(schema_name=schema_name, report_path=report_path, compression=compression) processor.process() stats_recorder.log_last_completed_datetime() stats_recorder.commit() stats_recorder.close_session() files = remove_files.remove_temp_cur_files(path.dirname(report_path)) LOG.info('Temporary files removed: %s', str(files))
def test_initializer_ocp(self): """Test to initializer for OCP.""" processor = ReportProcessor( schema_name=self.schema, report_path= ('./koku/masu/test/data/ocp/' 'e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv' ), compression='PLAIN', provider=OPENSHIFT_CONTAINER_PLATFORM, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) self.assertIsNotNone(processor._processor)
def test_initializer_ocp(self): """Test to initializer for OCP.""" processor = ReportProcessor( schema_name=self.schema, report_path= ("./koku/masu/test/data/ocp/" "e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv" ), compression="PLAIN", provider=Provider.PROVIDER_OCP, provider_uuid=self.aws_provider_uuid, manifest_id=None, ) self.assertIsNotNone(processor._processor)
def process_report(self, report, compression, provider_type, provider, manifest, bill_date=None): """Run the report processor on a report.""" status = baker.make("CostUsageReportStatus", manifest=manifest, report_name=report) status.last_started_datetime = self.dh.now context = {"start_date": bill_date, "tracing_id": uuid4()} ReportProcessor(self.schema, report, compression, provider_type, provider.uuid, manifest.id, context=context).process() status.last_completed_datetime = self.dh.now status.save()