def test_azure_process_camelCase(self): """Test the processing of an uncompressed Azure V2 file.""" counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_camelCase_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs("masu.processor.azure.azure_report_processor", level="INFO") as logger: processor.process() self.assertIn("INFO:masu.processor.azure.azure_report_processor", logger.output[0]) self.assertIn("azure_camelCased.csv", logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count >= counts[table_name]) self.assertFalse(os.path.exists(self.test_camelCase_report))
def notest_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1): counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs( 'masu.processor.azure.azure_report_processor', level='INFO' ) as logger: processor.process() self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0]) self.assertIn( 'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0] ) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name])
def test_azure_process_duplicates(self): """Test that row duplicates are not inserted into the DB.""" counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema with schema_context(self.schema): table_name = AZURE_REPORT_TABLE_MAP["line_item"] table = getattr(report_schema, table_name) initial_line_item_count = table.objects.count() # Process for the first time processor.process() for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count # Wipe stale data with schema_context(self.schema): self.accessor._get_db_obj_query( AZURE_REPORT_TABLE_MAP["line_item"]).delete() shutil.copy2(self.test_report_path, self.test_report) processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the second time processor.process() for table_name in self.report_tables: with self.subTest(table_name=table_name): table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() if table_name == AZURE_REPORT_TABLE_MAP["line_item"]: counts[table_name] = counts[ table_name] - initial_line_item_count print(count, counts[table_name], table_name) self.assertTrue(count == counts[table_name])
def test_azure_process_can_run_twice(self): """Test that row duplicates are inserted into the DB when process called twice.""" processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the first time processor.process() shutil.copy2(self.test_report_path, self.test_report) try: processor.process() except InternalError: self.fail("failed to call process twice.")
class AzureReportProcessorTest(MasuTestCase): """Test Cases for the AzureReportProcessor object.""" @classmethod def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.test_report_path = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv" cls.test_v2_report_path = "./koku/masu/test/data/azure/azure_version_2.csv" cls.camelCase_report_path = "./koku/masu/test/data/azure/azure_camelCased.csv" cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor() _report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP) _report_tables.pop("line_item_daily_summary", None) _report_tables.pop("tags_summary", None) _report_tables.pop("enabled_tag_keys", None) _report_tables.pop("ocp_on_azure_daily_summary", None) _report_tables.pop("ocp_on_azure_project_daily_summary", None) _report_tables.pop("ocp_on_azure_tags_summary", None) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report_path, "r", encoding="utf-8-sig") as f: reader = csv.DictReader(f) cls.row = next(reader) cls.row = {key.lower(): value for key, value in cls.row.items()} def setUp(self): """Set up each test.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f"{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv" self.test_v2_report = f"{self.temp_dir}/azure_version_2.csv" self.test_camelCase_report = f"{self.temp_dir}/azure_camelCased.csv" shutil.copy2(self.test_report_path, self.test_report) shutil.copy2(self.test_v2_report_path, self.test_v2_report) shutil.copy2(self.camelCase_report_path, self.test_camelCase_report) self.assembly_id = "1234" self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) self.processor_v2 = AzureReportProcessor( schema_name=self.schema, report_path=self.test_v2_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) billing_start = self.date_accessor.today_with_timezone("UTC").replace( year=2018, month=6, day=1, hour=0, minute=0, second=0) self.assembly_id = "1234" self.manifest_dict = { "assembly_id": self.assembly_id, "billing_period_start_datetime": billing_start, "num_total_files": 1, "provider_uuid": self.azure_provider_uuid, } self.accessor = AzureReportDBAccessor(self.schema) self.report_schema = self.accessor.report_schema self.manifest = self.manifest_accessor.add(**self.manifest_dict) def tearDown(self): """Tear down test case.""" super().tearDown() shutil.rmtree(self.temp_dir) def test_azure_initializer(self): """Test Azure initializer.""" self.assertIsNotNone(self.processor._schema) self.assertIsNotNone(self.processor._report_path) self.assertIsNotNone(self.processor._report_name) self.assertIsNotNone(self.processor._compression) self.assertEqual(self.processor._datetime_format, Config.AZURE_DATETIME_STR_FORMAT) self.assertEqual(self.processor._batch_size, Config.REPORT_PROCESSING_BATCH_SIZE) def test_azure_process(self): """Test the processing of an uncompressed Azure file.""" counts = {} report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs("masu.processor.azure.azure_report_processor", level="INFO") as logger: self.processor.process() self.assertIn("INFO:masu.processor.azure.azure_report_processor", logger.output[0]) self.assertIn( "costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv", logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name]) self.assertFalse(os.path.exists(self.test_report)) def test_azure_process_v2(self): """Test the processing of an uncompressed Azure V2 file.""" counts = {} report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs("masu.processor.azure.azure_report_processor", level="INFO") as logger: self.processor_v2.process() self.assertIn("INFO:masu.processor.azure.azure_report_processor", logger.output[0]) self.assertIn("azure_version_2.csv", logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count >= counts[table_name]) self.assertFalse(os.path.exists(self.test_v2_report)) def test_azure_process_camelCase(self): """Test the processing of an uncompressed Azure V2 file.""" counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_camelCase_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs("masu.processor.azure.azure_report_processor", level="INFO") as logger: processor.process() self.assertIn("INFO:masu.processor.azure.azure_report_processor", logger.output[0]) self.assertIn("azure_camelCased.csv", logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count >= counts[table_name]) self.assertFalse(os.path.exists(self.test_camelCase_report)) def test_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, "REPORT_PROCESSING_BATCH_SIZE", 2): # Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 2. self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Re-run test with new configuration and verify it's still successful. self.test_azure_process() def notest_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, "REPORT_PROCESSING_BATCH_SIZE", 1): counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs("masu.processor.azure.azure_report_processor", level="INFO") as logger: processor.process() self.assertIn( "INFO:masu.processor.azure.azure_report_processor", logger.output[0]) self.assertIn( "costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv", logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name]) def test_azure_process_duplicates(self): """Test that row duplicates are not inserted into the DB.""" counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema with schema_context(self.schema): table_name = AZURE_REPORT_TABLE_MAP["line_item"] table = getattr(report_schema, table_name) initial_line_item_count = table.objects.count() # Process for the first time processor.process() for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count # Wipe stale data with schema_context(self.schema): self.accessor._get_db_obj_query( AZURE_REPORT_TABLE_MAP["line_item"]).delete() shutil.copy2(self.test_report_path, self.test_report) processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the second time processor.process() for table_name in self.report_tables: with self.subTest(table_name=table_name): table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() if table_name == AZURE_REPORT_TABLE_MAP["line_item"]: counts[table_name] = counts[ table_name] - initial_line_item_count print(count, counts[table_name], table_name) self.assertTrue(count == counts[table_name]) def test_azure_process_can_run_twice(self): """Test that row duplicates are inserted into the DB when process called twice.""" processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the first time processor.process() shutil.copy2(self.test_report_path, self.test_report) try: processor.process() except InternalError: self.fail("failed to call process twice.") def test_azure_create_cost_entry_bill(self): """Test that a cost entry bill id is returned.""" table_name = AZURE_REPORT_TABLE_MAP["bill"] bill_id = self.processor._create_cost_entry_bill( self.row, self.accessor) self.assertIsNotNone(bill_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by("-id").first().id provider_uuid = query.order_by("-id").first().provider_id self.assertEqual(bill_id, id_in_db) self.assertIsNotNone(provider_uuid) def test_azure_create_product(self): """Test that a product id is returned.""" table_name = AZURE_REPORT_TABLE_MAP["product"] product_id = self.processor._create_cost_entry_product( self.row, self.accessor) self.assertIsNotNone(product_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by("-id").first().id self.assertEqual(product_id, id_in_db) def test_azure_create_meter(self): """Test that a meter id is returned.""" table_name = AZURE_REPORT_TABLE_MAP["meter"] meter_id = self.processor._create_meter(self.row, self.accessor) self.assertIsNotNone(meter_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by("-id").first().id self.assertEqual(meter_id, id_in_db) def test_azure_create_cost_entry_line_item(self): """Test that Azure line item data is returned properly.""" bill_id = self.processor._create_cost_entry_bill( self.row, self.accessor) product_id = self.processor._create_cost_entry_product( self.row, self.accessor) meter_id = self.processor._create_meter(self.row, self.accessor) self.processor._create_cost_entry_line_item(self.row, bill_id, product_id, meter_id, self.accessor) line_item = None if self.processor.processed_report.line_items: line_item = self.processor.processed_report.line_items[-1] self.assertIsNotNone(line_item) self.assertIn("tags", line_item) self.assertEqual(line_item.get("cost_entry_bill_id"), bill_id) self.assertEqual(line_item.get("cost_entry_product_id"), product_id) self.assertEqual(line_item.get("meter_id"), meter_id) self.assertIsNotNone(self.processor.line_item_columns) def test_should_process_row_within_cuttoff_date(self): """Test that we correctly determine a row should be processed.""" today = self.date_accessor.today_with_timezone("UTC") row = {"usagedatetime": today.isoformat()} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) should_process = processor._should_process_row(row, "usagedatetime", False) self.assertTrue(should_process) def test_should_process_row_outside_cuttoff_date(self): """Test that we correctly determine a row should be processed.""" today = self.date_accessor.today_with_timezone("UTC") usage_start = today - relativedelta(days=10) row = {"usagedatetime": usage_start.isoformat()} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) should_process = processor._should_process_row(row, "usagedatetime", False) self.assertFalse(should_process) def test_get_date_column_filter(self): """Test that the Azure specific filter is returned.""" processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) date_filter = processor.get_date_column_filter() self.assertIn("usage_date__gte", date_filter) def test_normalize_header(self): """Test that headers are always converted to English.""" english_header = ( "DepartmentName,AccountName,AccountOwnerId,SubscriptionGuid," "SubscriptionName,ResourceGroup,ResourceLocation,UsageDateTime," "ProductName,MeterCategory,MeterSubcategory,MeterId,MeterName," "MeterRegion,UnitOfMeasure,UsageQuantity,ResourceRate,PreTaxCost," "CostCenter,ConsumedService,ResourceType,InstanceId,Tags,OfferId," "AdditionalInfo,ServiceInfo1,ServiceInfo2") spanish_header = ( "Nombre de departamento (DepartmentName),Nombre de cuenta (AccountName)," "Id. del propietario de la cuenta (AccountOwnerId),Ubicación de los recursos (SubscriptionGuid)," "Nombre de la suscripción (SubscriptionName),ResourceGroup (ResourceGroup)," "Ubicación de los recursos (ResourceLocation),UsageDateTime (UsageDateTime)," "ProductName (ProductName),Categoría del medidor (MeterCategory)," "MeterSubcategory (MeterSubcategory),Id. de medidor (MeterId)," "Nombre del medidor (MeterName),Región del medidor (MeterRegion)," "Unidad de medida (UnitOfMeasure),UsageQuantity (UsageQuantity)," "Tasa de recursos (ResourceRate),PreTaxCost (PreTaxCost)," "Centro de coste (CostCenter),Servicio consumido (ConsumedService)," "ResourceType (ResourceType),InstanceId (InstanceId),Etiquetas (Tags)," "OfferId (OfferId),Información adicional (AdditionalInfo)," "Información del servicio 1 (ServiceInfo1)," "Información del servicio 2 (ServiceInfo2)") expected_header = [ "departmentname", "accountname", "accountownerid", "subscriptionguid", "subscriptionname", "resourcegroup", "resourcelocation", "usagedatetime", "productname", "metercategory", "metersubcategory", "meterid", "metername", "meterregion", "unitofmeasure", "usagequantity", "resourcerate", "pretaxcost", "costcenter", "consumedservice", "resourcetype", "instanceid", "tags", "offerid", "additionalinfo", "serviceinfo1", "serviceinfo2", ] self.assertEqual(normalize_header(english_header), expected_header) self.assertEqual(normalize_header(spanish_header), expected_header) def test_process_tags(self): """Test that both tag string formats are parsed properly.""" processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) tag_str_as_json = '{"project":"p1","cost":"management"}' tag_str_no_json = '"project": "p1","cost": "management"' expected_tag_str = '{"project":"p1","cost":"management"}' self.assertEqual(processor._process_tags(tag_str_as_json), expected_tag_str) self.assertEqual(processor._process_tags(tag_str_no_json), expected_tag_str) self.assertEqual(processor._process_tags(""), "{}")
class AzureReportProcessorTest(MasuTestCase): """Test Cases for the AzureReportProcessor object.""" @classmethod def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv' cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map _report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP) _report_tables.pop('line_item_daily_summary', None) _report_tables.pop('tags_summary', None) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report, 'r', encoding='utf-8-sig') as f: reader = csv.DictReader(f) cls.row = next(reader) def setUp(self): """Set up each test.""" super().setUp() self.assembly_id = '1234' self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_id=self.azure_provider_id, ) billing_start = self.date_accessor.today_with_timezone('UTC').replace( year=2018, month=6, day=1, hour=0, minute=0, second=0) self.assembly_id = '1234' self.manifest_dict = { 'assembly_id': self.assembly_id, 'billing_period_start_datetime': billing_start, 'num_total_files': 1, 'provider_id': self.azure_provider.id, } self.accessor = AzureReportDBAccessor(self.schema, self.column_map) self.report_schema = self.accessor.report_schema self.manifest = self.manifest_accessor.add(**self.manifest_dict) self.manifest_accessor.commit() def test_azure_initializer(self): """Test Azure initializer.""" self.assertIsNotNone(self.processor._schema_name) self.assertIsNotNone(self.processor._report_path) self.assertIsNotNone(self.processor._report_name) self.assertIsNotNone(self.processor._compression) self.assertEqual(self.processor._datetime_format, Config.AZURE_DATETIME_STR_FORMAT) self.assertEqual(self.processor._batch_size, Config.REPORT_PROCESSING_BATCH_SIZE) def test_azure_process(self): """Test the processing of an uncompressed Azure file.""" counts = {} report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs('masu.processor.azure.azure_report_processor', level='INFO') as logger: self.processor.process() self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0]) self.assertIn( 'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name]) def test_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1): # Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 1. self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_id=self.azure_provider_id, ) # Re-run test with new configuration and verify it's still successful. self.test_azure_process() def notest_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1): counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_id=self.azure_provider_id, ) report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs('masu.processor.azure.azure_report_processor', level='INFO') as logger: processor.process() self.assertIn( 'INFO:masu.processor.azure.azure_report_processor', logger.output[0]) self.assertIn( 'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name]) def test_azure_process_duplicates(self): """Test that row duplicates are not inserted into the DB.""" counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_id=self.azure_provider.id, ) # Process for the first time processor.process() report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count # Wipe stale data with schema_context(self.schema): self.accessor._get_db_obj_query( AZURE_REPORT_TABLE_MAP['line_item']).delete() processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_id=self.azure_provider.id, ) # Process for the second time processor.process() for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count == counts[table_name]) def test_azure_create_cost_entry_bill(self): """Test that a cost entry bill id is returned.""" table_name = AZURE_REPORT_TABLE_MAP['bill'] table = getattr(self.report_schema, table_name) bill_id = self.processor._create_cost_entry_bill( self.row, self.accessor) self.assertIsNotNone(bill_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by('-id').first().id provider_id = query.order_by('-id').first().provider_id self.assertEqual(bill_id, id_in_db) self.assertIsNotNone(provider_id) def test_azure_create_product(self): """Test that a product id is returned.""" table_name = AZURE_REPORT_TABLE_MAP['product'] table = getattr(self.report_schema, table_name) product_id = self.processor._create_cost_entry_product( self.row, self.accessor) self.assertIsNotNone(product_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by('-id').first().id self.assertEqual(product_id, id_in_db) def test_azure_create_meter(self): """Test that a meter id is returned.""" table_name = AZURE_REPORT_TABLE_MAP['meter'] table = getattr(self.report_schema, table_name) meter_id = self.processor._create_meter(self.row, self.accessor) self.assertIsNotNone(meter_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by('-id').first().id self.assertEqual(meter_id, id_in_db) def test_azure_create_cost_entry_line_item(self): """Test that Azure line item data is returned properly.""" bill_id = self.processor._create_cost_entry_bill( self.row, self.accessor) product_id = self.processor._create_cost_entry_product( self.row, self.accessor) meter_id = self.processor._create_meter(self.row, self.accessor) self.accessor.commit() self.processor._create_cost_entry_line_item( self.row, bill_id, product_id, meter_id, self.accessor, ) line_item = None if self.processor.processed_report.line_items: line_item = self.processor.processed_report.line_items[-1] self.assertIsNotNone(line_item) self.assertIn('tags', line_item) self.assertEqual(line_item.get('cost_entry_bill_id'), bill_id) self.assertEqual(line_item.get('cost_entry_product_id'), product_id) self.assertEqual(line_item.get('meter_id'), meter_id) self.assertIsNotNone(self.processor.line_item_columns) def test_azure_remove_temp_cur_files(self): """Test to remove temporary cost usage files.""" cur_dir = tempfile.mkdtemp() manifest_data = {"assemblyId": "31727a10-f4b4-43a2-80e5-bef1aaeabfc1"} manifest = '{}/{}'.format(cur_dir, 'Manifest.json') with open(manifest, 'w') as outfile: json.dump(manifest_data, outfile) file_list = [ { 'file': 'costreport_31727a10-f4b4-43a2-80e5-bef1aaeabfc1.csv', 'processed_date': datetime.datetime(year=2018, month=5, day=3), }, { 'file': 'costreport_31727a10-f4b4-43a2-80e5-bef1aaeabfc1.csv', 'processed_date': datetime.datetime(year=2018, month=5, day=3), }, { 'file': 'costreport_2aeb9169-2526-441c-9eca-d7ed015d52bd.csv', 'processed_date': datetime.datetime(year=2018, month=5, day=2), }, { 'file': 'costreport_6c8487e8-c590-4e6a-b2c2-91a2375c0bad.csv', 'processed_date': datetime.datetime(year=2018, month=5, day=1), }, { 'file': 'costreport_6c8487e8-c590-4e6a-b2c2-91a2375d0bed.csv', 'processed_date': None, }, ] expected_delete_list = [] for item in file_list: path = '{}/{}'.format(cur_dir, item['file']) f = open(path, 'w') obj = self.manifest_accessor.get_manifest(self.assembly_id, self.azure_provider.id) with ReportStatsDBAccessor(item['file'], obj.id) as stats: stats.update(last_completed_datetime=item['processed_date']) f.close() if (not manifest_data.get('assemblyId') in item['file']): expected_delete_list.append(path) removed_files = self.processor.remove_temp_cur_files(cur_dir) self.assertEqual(sorted(removed_files), sorted(expected_delete_list)) shutil.rmtree(cur_dir)
class AzureReportProcessorTest(MasuTestCase): """Test Cases for the AzureReportProcessor object.""" @classmethod def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.test_report_path = ( './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv' ) cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map _report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP) _report_tables.pop('line_item_daily_summary', None) _report_tables.pop('tags_summary', None) _report_tables.pop('ocp_on_azure_daily_summary', None) _report_tables.pop('ocp_on_azure_project_daily_summary', None) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report_path, 'r', encoding='utf-8-sig') as f: reader = csv.DictReader(f) cls.row = next(reader) def setUp(self): """Set up each test.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f'{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv' shutil.copy2(self.test_report_path, self.test_report) self.assembly_id = '1234' self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) billing_start = self.date_accessor.today_with_timezone('UTC').replace( year=2018, month=6, day=1, hour=0, minute=0, second=0 ) self.assembly_id = '1234' self.manifest_dict = { 'assembly_id': self.assembly_id, 'billing_period_start_datetime': billing_start, 'num_total_files': 1, 'provider_uuid': self.azure_provider_uuid, } self.accessor = AzureReportDBAccessor(self.schema, self.column_map) self.report_schema = self.accessor.report_schema self.manifest = self.manifest_accessor.add(**self.manifest_dict) def tearDown(self): """Tear down test case.""" super().tearDown() shutil.rmtree(self.temp_dir) def test_azure_initializer(self): """Test Azure initializer.""" self.assertIsNotNone(self.processor._schema) self.assertIsNotNone(self.processor._report_path) self.assertIsNotNone(self.processor._report_name) self.assertIsNotNone(self.processor._compression) self.assertEqual(self.processor._datetime_format, Config.AZURE_DATETIME_STR_FORMAT) self.assertEqual(self.processor._batch_size, Config.REPORT_PROCESSING_BATCH_SIZE) def test_azure_process(self): """Test the processing of an uncompressed Azure file.""" counts = {} report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs('masu.processor.azure.azure_report_processor', level='INFO') as logger: self.processor.process() self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0]) self.assertIn('costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0]) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name]) self.assertFalse(os.path.exists(self.test_report)) def test_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1): # Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 1. self.processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Re-run test with new configuration and verify it's still successful. self.test_azure_process() def notest_process_azure_small_batches(self): """Test the processing of an uncompressed azure file in small batches.""" with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1): counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count logging.disable( logging.NOTSET ) # We are currently disabling all logging below CRITICAL in masu/__init__.py with self.assertLogs( 'masu.processor.azure.azure_report_processor', level='INFO' ) as logger: processor.process() self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0]) self.assertIn( 'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0] ) for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count > counts[table_name]) def test_azure_process_duplicates(self): """Test that row duplicates are not inserted into the DB.""" counts = {} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the first time processor.process() report_db = self.accessor report_schema = report_db.report_schema for table_name in self.report_tables: table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() counts[table_name] = count # Wipe stale data with schema_context(self.schema): self.accessor._get_db_obj_query(AZURE_REPORT_TABLE_MAP['line_item']).delete() shutil.copy2(self.test_report_path, self.test_report) processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the second time processor.process() for table_name in self.report_tables: with self.subTest(table_name=table_name): table = getattr(report_schema, table_name) with schema_context(self.schema): count = table.objects.count() self.assertTrue(count == counts[table_name]) def test_azure_process_can_run_twice(self): """Test that row duplicates are inserted into the DB when process called twice.""" processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) # Process for the first time processor.process() shutil.copy2(self.test_report_path, self.test_report) try: processor.process() except InternalError: self.fail('failed to call process twice.') def test_azure_create_cost_entry_bill(self): """Test that a cost entry bill id is returned.""" table_name = AZURE_REPORT_TABLE_MAP['bill'] bill_id = self.processor._create_cost_entry_bill(self.row, self.accessor) self.assertIsNotNone(bill_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by('-id').first().id provider_uuid = query.order_by('-id').first().provider_id self.assertEqual(bill_id, id_in_db) self.assertIsNotNone(provider_uuid) def test_azure_create_product(self): """Test that a product id is returned.""" table_name = AZURE_REPORT_TABLE_MAP['product'] product_id = self.processor._create_cost_entry_product(self.row, self.accessor) self.assertIsNotNone(product_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by('-id').first().id self.assertEqual(product_id, id_in_db) def test_azure_create_meter(self): """Test that a meter id is returned.""" table_name = AZURE_REPORT_TABLE_MAP['meter'] meter_id = self.processor._create_meter(self.row, self.accessor) self.assertIsNotNone(meter_id) query = self.accessor._get_db_obj_query(table_name) id_in_db = query.order_by('-id').first().id self.assertEqual(meter_id, id_in_db) def test_azure_create_cost_entry_line_item(self): """Test that Azure line item data is returned properly.""" bill_id = self.processor._create_cost_entry_bill(self.row, self.accessor) product_id = self.processor._create_cost_entry_product(self.row, self.accessor) meter_id = self.processor._create_meter(self.row, self.accessor) self.processor._create_cost_entry_line_item( self.row, bill_id, product_id, meter_id, self.accessor, ) line_item = None if self.processor.processed_report.line_items: line_item = self.processor.processed_report.line_items[-1] self.assertIsNotNone(line_item) self.assertIn('tags', line_item) self.assertEqual(line_item.get('cost_entry_bill_id'), bill_id) self.assertEqual(line_item.get('cost_entry_product_id'), product_id) self.assertEqual(line_item.get('meter_id'), meter_id) self.assertIsNotNone(self.processor.line_item_columns) def test_should_process_row_within_cuttoff_date(self): """Test that we correctly determine a row should be processed.""" today = self.date_accessor.today_with_timezone('UTC') row = {'UsageDateTime': today.isoformat()} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) should_process = processor._should_process_row(row, 'UsageDateTime', False) self.assertTrue(should_process) def test_should_process_row_outside_cuttoff_date(self): """Test that we correctly determine a row should be processed.""" today = self.date_accessor.today_with_timezone('UTC') usage_start = today - relativedelta(days=10) row = {'UsageDateTime': usage_start.isoformat()} processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) should_process = processor._should_process_row(row, 'UsageDateTime', False) self.assertFalse(should_process) def test_get_date_column_filter(self): """Test that the Azure specific filter is returned.""" processor = AzureReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.azure_provider_uuid, ) date_filter = processor.get_date_column_filter() self.assertIn('usage_date_time__gte', date_filter)