def setUp(self):
"""Set up each test."""
super().setUp()
self.assembly_id = '1234'
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider_id,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 1,
'provider_id': self.azure_provider.id,
}
self.accessor = AzureReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
def setUp(self):
"""Set up each test."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f'{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
shutil.copy2(self.test_report_path, self.test_report)
self.assembly_id = '1234'
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0
)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 1,
'provider_uuid': self.azure_provider_uuid,
}
self.accessor = AzureReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def notest_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1):
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs(
'masu.processor.azure.azure_report_processor', level='INFO'
) as logger:
processor.process()
self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0])
self.assertIn(
'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0]
)
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
def test_azure_process_camelCase(self):
"""Test the processing of an uncompressed Azure V2 file."""
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_camelCase_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs("masu.processor.azure.azure_report_processor",
level="INFO") as logger:
processor.process()
self.assertIn("INFO:masu.processor.azure.azure_report_processor",
logger.output[0])
self.assertIn("azure_camelCased.csv", logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count >= counts[table_name])
self.assertFalse(os.path.exists(self.test_camelCase_report))
def test_get_date_column_filter(self):
"""Test that the Azure specific filter is returned."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
date_filter = processor.get_date_column_filter()
self.assertIn("usage_date__gte", date_filter)
def test_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, "REPORT_PROCESSING_BATCH_SIZE", 2):
# Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 2.
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Re-run test with new configuration and verify it's still successful.
self.test_azure_process()
def test_should_process_row_within_cuttoff_date(self):
"""Test that we correctly determine a row should be processed."""
today = self.date_accessor.today_with_timezone('UTC')
row = {'UsageDateTime': today.isoformat()}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
should_process = processor._should_process_row(row, 'UsageDateTime', False)
self.assertTrue(should_process)
def test_should_process_row_outside_cuttoff_date(self):
"""Test that we correctly determine a row should be processed."""
today = self.date_accessor.today_with_timezone("UTC")
usage_start = today - relativedelta(days=10)
row = {"UsageDateTime": usage_start.isoformat()}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
should_process = processor._should_process_row(row, "UsageDateTime", False)
self.assertFalse(should_process)
def _set_processor(self):
"""
Create the report processor object.
Processor is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Provider-specific report processor
"""
if self.provider_type in (AMAZON_WEB_SERVICES,
AWS_LOCAL_SERVICE_PROVIDER):
return AWSReportProcessor(schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_id=self.provider_id,
manifest_id=self.manifest_id)
if self.provider_type in (AZURE, AZURE_LOCAL_SERVICE_PROVIDER):
return AzureReportProcessor(schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_id=self.provider_id,
manifest_id=self.manifest_id)
if self.provider_type in (OPENSHIFT_CONTAINER_PLATFORM, ):
return OCPReportProcessor(schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_id=self.provider_id)
return None
def _set_processor(self):
"""
Create the report processor object.
Processor is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Provider-specific report processor
"""
if enable_trino_processing(self.provider_uuid):
return ParquetReportProcessor(
schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_uuid=self.provider_uuid,
provider_type=self.provider_type,
manifest_id=self.manifest_id,
context=self.context,
)
if self.provider_type in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
return AWSReportProcessor(
schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_uuid=self.provider_uuid,
manifest_id=self.manifest_id,
)
if self.provider_type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
return AzureReportProcessor(
schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_uuid=self.provider_uuid,
manifest_id=self.manifest_id,
)
if self.provider_type in (Provider.PROVIDER_OCP, ):
return OCPReportProcessor(
schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_uuid=self.provider_uuid,
)
if self.provider_type in (Provider.PROVIDER_GCP,
Provider.PROVIDER_GCP_LOCAL):
return GCPReportProcessor(
schema_name=self.schema_name,
report_path=self.report_path,
compression=self.compression,
provider_uuid=self.provider_uuid,
manifest_id=self.manifest_id,
)
return None
def test_process_tags(self):
"""Test that both tag string formats are parsed properly."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
tag_str_as_json = '{"project":"p1","cost":"management"}'
tag_str_no_json = '"project": "p1","cost": "management"'
expected_tag_str = '{"project":"p1","cost":"management"}'
self.assertEqual(processor._process_tags(tag_str_as_json),
expected_tag_str)
self.assertEqual(processor._process_tags(tag_str_no_json),
expected_tag_str)
self.assertEqual(processor._process_tags(""), "{}")
def test_azure_process_duplicates(self):
"""Test that row duplicates are not inserted into the DB."""
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
with schema_context(self.schema):
table_name = AZURE_REPORT_TABLE_MAP["line_item"]
table = getattr(report_schema, table_name)
initial_line_item_count = table.objects.count()
# Process for the first time
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
# Wipe stale data
with schema_context(self.schema):
self.accessor._get_db_obj_query(
AZURE_REPORT_TABLE_MAP["line_item"]).delete()
shutil.copy2(self.test_report_path, self.test_report)
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the second time
processor.process()
for table_name in self.report_tables:
with self.subTest(table_name=table_name):
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
if table_name == AZURE_REPORT_TABLE_MAP["line_item"]:
counts[table_name] = counts[
table_name] - initial_line_item_count
print(count, counts[table_name], table_name)
self.assertTrue(count == counts[table_name])
def setUp(self):
"""Set up each test."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f"{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
self.test_v2_report = f"{self.temp_dir}/azure_version_2.csv"
self.test_camelCase_report = f"{self.temp_dir}/azure_camelCased.csv"
shutil.copy2(self.test_report_path, self.test_report)
shutil.copy2(self.test_v2_report_path, self.test_v2_report)
shutil.copy2(self.camelCase_report_path, self.test_camelCase_report)
self.assembly_id = "1234"
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
self.processor_v2 = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_v2_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
billing_start = self.date_accessor.today_with_timezone("UTC").replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
self.assembly_id = "1234"
self.manifest_dict = {
"assembly_id": self.assembly_id,
"billing_period_start_datetime": billing_start,
"num_total_files": 1,
"provider_uuid": self.azure_provider_uuid,
}
self.accessor = AzureReportDBAccessor(self.schema)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def test_azure_process_can_run_twice(self):
"""Test that row duplicates are inserted into the DB when process called twice."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the first time
processor.process()
shutil.copy2(self.test_report_path, self.test_report)
try:
processor.process()
except InternalError:
self.fail("failed to call process twice.")
class AzureReportProcessorTest(MasuTestCase):
"""Test Cases for the AzureReportProcessor object."""
@classmethod
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report_path = (
'./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
)
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
_report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP)
_report_tables.pop('line_item_daily_summary', None)
_report_tables.pop('tags_summary', None)
_report_tables.pop('ocp_on_azure_daily_summary', None)
_report_tables.pop('ocp_on_azure_project_daily_summary', None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_path, 'r', encoding='utf-8-sig') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def setUp(self):
"""Set up each test."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f'{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
shutil.copy2(self.test_report_path, self.test_report)
self.assembly_id = '1234'
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0
)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 1,
'provider_uuid': self.azure_provider_uuid,
}
self.accessor = AzureReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def tearDown(self):
"""Tear down test case."""
super().tearDown()
shutil.rmtree(self.temp_dir)
def test_azure_initializer(self):
"""Test Azure initializer."""
self.assertIsNotNone(self.processor._schema)
self.assertIsNotNone(self.processor._report_path)
self.assertIsNotNone(self.processor._report_name)
self.assertIsNotNone(self.processor._compression)
self.assertEqual(self.processor._datetime_format, Config.AZURE_DATETIME_STR_FORMAT)
self.assertEqual(self.processor._batch_size, Config.REPORT_PROCESSING_BATCH_SIZE)
def test_azure_process(self):
"""Test the processing of an uncompressed Azure file."""
counts = {}
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs('masu.processor.azure.azure_report_processor', level='INFO') as logger:
self.processor.process()
self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0])
self.assertIn('costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
self.assertFalse(os.path.exists(self.test_report))
def test_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1):
# Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 1.
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Re-run test with new configuration and verify it's still successful.
self.test_azure_process()
def notest_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1):
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs(
'masu.processor.azure.azure_report_processor', level='INFO'
) as logger:
processor.process()
self.assertIn('INFO:masu.processor.azure.azure_report_processor', logger.output[0])
self.assertIn(
'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv', logger.output[0]
)
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
def test_azure_process_duplicates(self):
"""Test that row duplicates are not inserted into the DB."""
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
# Wipe stale data
with schema_context(self.schema):
self.accessor._get_db_obj_query(AZURE_REPORT_TABLE_MAP['line_item']).delete()
shutil.copy2(self.test_report_path, self.test_report)
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the second time
processor.process()
for table_name in self.report_tables:
with self.subTest(table_name=table_name):
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count == counts[table_name])
def test_azure_process_can_run_twice(self):
"""Test that row duplicates are inserted into the DB when process called twice."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the first time
processor.process()
shutil.copy2(self.test_report_path, self.test_report)
try:
processor.process()
except InternalError:
self.fail('failed to call process twice.')
def test_azure_create_cost_entry_bill(self):
"""Test that a cost entry bill id is returned."""
table_name = AZURE_REPORT_TABLE_MAP['bill']
bill_id = self.processor._create_cost_entry_bill(self.row, self.accessor)
self.assertIsNotNone(bill_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by('-id').first().id
provider_uuid = query.order_by('-id').first().provider_id
self.assertEqual(bill_id, id_in_db)
self.assertIsNotNone(provider_uuid)
def test_azure_create_product(self):
"""Test that a product id is returned."""
table_name = AZURE_REPORT_TABLE_MAP['product']
product_id = self.processor._create_cost_entry_product(self.row, self.accessor)
self.assertIsNotNone(product_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by('-id').first().id
self.assertEqual(product_id, id_in_db)
def test_azure_create_meter(self):
"""Test that a meter id is returned."""
table_name = AZURE_REPORT_TABLE_MAP['meter']
meter_id = self.processor._create_meter(self.row, self.accessor)
self.assertIsNotNone(meter_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by('-id').first().id
self.assertEqual(meter_id, id_in_db)
def test_azure_create_cost_entry_line_item(self):
"""Test that Azure line item data is returned properly."""
bill_id = self.processor._create_cost_entry_bill(self.row, self.accessor)
product_id = self.processor._create_cost_entry_product(self.row, self.accessor)
meter_id = self.processor._create_meter(self.row, self.accessor)
self.processor._create_cost_entry_line_item(
self.row, bill_id, product_id, meter_id, self.accessor,
)
line_item = None
if self.processor.processed_report.line_items:
line_item = self.processor.processed_report.line_items[-1]
self.assertIsNotNone(line_item)
self.assertIn('tags', line_item)
self.assertEqual(line_item.get('cost_entry_bill_id'), bill_id)
self.assertEqual(line_item.get('cost_entry_product_id'), product_id)
self.assertEqual(line_item.get('meter_id'), meter_id)
self.assertIsNotNone(self.processor.line_item_columns)
def test_should_process_row_within_cuttoff_date(self):
"""Test that we correctly determine a row should be processed."""
today = self.date_accessor.today_with_timezone('UTC')
row = {'UsageDateTime': today.isoformat()}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
should_process = processor._should_process_row(row, 'UsageDateTime', False)
self.assertTrue(should_process)
def test_should_process_row_outside_cuttoff_date(self):
"""Test that we correctly determine a row should be processed."""
today = self.date_accessor.today_with_timezone('UTC')
usage_start = today - relativedelta(days=10)
row = {'UsageDateTime': usage_start.isoformat()}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
should_process = processor._should_process_row(row, 'UsageDateTime', False)
self.assertFalse(should_process)
def test_get_date_column_filter(self):
"""Test that the Azure specific filter is returned."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
date_filter = processor.get_date_column_filter()
self.assertIn('usage_date_time__gte', date_filter)
class AzureReportProcessorTest(MasuTestCase):
"""Test Cases for the AzureReportProcessor object."""
@classmethod
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report_path = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
cls.test_v2_report_path = "./koku/masu/test/data/azure/azure_version_2.csv"
cls.camelCase_report_path = "./koku/masu/test/data/azure/azure_camelCased.csv"
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
_report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP)
_report_tables.pop("line_item_daily_summary", None)
_report_tables.pop("tags_summary", None)
_report_tables.pop("enabled_tag_keys", None)
_report_tables.pop("ocp_on_azure_daily_summary", None)
_report_tables.pop("ocp_on_azure_project_daily_summary", None)
_report_tables.pop("ocp_on_azure_tags_summary", None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_path, "r", encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
cls.row = next(reader)
cls.row = {key.lower(): value for key, value in cls.row.items()}
def setUp(self):
"""Set up each test."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f"{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
self.test_v2_report = f"{self.temp_dir}/azure_version_2.csv"
self.test_camelCase_report = f"{self.temp_dir}/azure_camelCased.csv"
shutil.copy2(self.test_report_path, self.test_report)
shutil.copy2(self.test_v2_report_path, self.test_v2_report)
shutil.copy2(self.camelCase_report_path, self.test_camelCase_report)
self.assembly_id = "1234"
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
self.processor_v2 = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_v2_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
billing_start = self.date_accessor.today_with_timezone("UTC").replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
self.assembly_id = "1234"
self.manifest_dict = {
"assembly_id": self.assembly_id,
"billing_period_start_datetime": billing_start,
"num_total_files": 1,
"provider_uuid": self.azure_provider_uuid,
}
self.accessor = AzureReportDBAccessor(self.schema)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def tearDown(self):
"""Tear down test case."""
super().tearDown()
shutil.rmtree(self.temp_dir)
def test_azure_initializer(self):
"""Test Azure initializer."""
self.assertIsNotNone(self.processor._schema)
self.assertIsNotNone(self.processor._report_path)
self.assertIsNotNone(self.processor._report_name)
self.assertIsNotNone(self.processor._compression)
self.assertEqual(self.processor._datetime_format,
Config.AZURE_DATETIME_STR_FORMAT)
self.assertEqual(self.processor._batch_size,
Config.REPORT_PROCESSING_BATCH_SIZE)
def test_azure_process(self):
"""Test the processing of an uncompressed Azure file."""
counts = {}
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs("masu.processor.azure.azure_report_processor",
level="INFO") as logger:
self.processor.process()
self.assertIn("INFO:masu.processor.azure.azure_report_processor",
logger.output[0])
self.assertIn(
"costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv",
logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
self.assertFalse(os.path.exists(self.test_report))
def test_azure_process_v2(self):
"""Test the processing of an uncompressed Azure V2 file."""
counts = {}
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs("masu.processor.azure.azure_report_processor",
level="INFO") as logger:
self.processor_v2.process()
self.assertIn("INFO:masu.processor.azure.azure_report_processor",
logger.output[0])
self.assertIn("azure_version_2.csv", logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count >= counts[table_name])
self.assertFalse(os.path.exists(self.test_v2_report))
def test_azure_process_camelCase(self):
"""Test the processing of an uncompressed Azure V2 file."""
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_camelCase_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs("masu.processor.azure.azure_report_processor",
level="INFO") as logger:
processor.process()
self.assertIn("INFO:masu.processor.azure.azure_report_processor",
logger.output[0])
self.assertIn("azure_camelCased.csv", logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count >= counts[table_name])
self.assertFalse(os.path.exists(self.test_camelCase_report))
def test_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, "REPORT_PROCESSING_BATCH_SIZE", 2):
# Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 2.
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Re-run test with new configuration and verify it's still successful.
self.test_azure_process()
def notest_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, "REPORT_PROCESSING_BATCH_SIZE", 1):
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs("masu.processor.azure.azure_report_processor",
level="INFO") as logger:
processor.process()
self.assertIn(
"INFO:masu.processor.azure.azure_report_processor",
logger.output[0])
self.assertIn(
"costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv",
logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
def test_azure_process_duplicates(self):
"""Test that row duplicates are not inserted into the DB."""
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
with schema_context(self.schema):
table_name = AZURE_REPORT_TABLE_MAP["line_item"]
table = getattr(report_schema, table_name)
initial_line_item_count = table.objects.count()
# Process for the first time
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
# Wipe stale data
with schema_context(self.schema):
self.accessor._get_db_obj_query(
AZURE_REPORT_TABLE_MAP["line_item"]).delete()
shutil.copy2(self.test_report_path, self.test_report)
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the second time
processor.process()
for table_name in self.report_tables:
with self.subTest(table_name=table_name):
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
if table_name == AZURE_REPORT_TABLE_MAP["line_item"]:
counts[table_name] = counts[
table_name] - initial_line_item_count
print(count, counts[table_name], table_name)
self.assertTrue(count == counts[table_name])
def test_azure_process_can_run_twice(self):
"""Test that row duplicates are inserted into the DB when process called twice."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
# Process for the first time
processor.process()
shutil.copy2(self.test_report_path, self.test_report)
try:
processor.process()
except InternalError:
self.fail("failed to call process twice.")
def test_azure_create_cost_entry_bill(self):
"""Test that a cost entry bill id is returned."""
table_name = AZURE_REPORT_TABLE_MAP["bill"]
bill_id = self.processor._create_cost_entry_bill(
self.row, self.accessor)
self.assertIsNotNone(bill_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by("-id").first().id
provider_uuid = query.order_by("-id").first().provider_id
self.assertEqual(bill_id, id_in_db)
self.assertIsNotNone(provider_uuid)
def test_azure_create_product(self):
"""Test that a product id is returned."""
table_name = AZURE_REPORT_TABLE_MAP["product"]
product_id = self.processor._create_cost_entry_product(
self.row, self.accessor)
self.assertIsNotNone(product_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by("-id").first().id
self.assertEqual(product_id, id_in_db)
def test_azure_create_meter(self):
"""Test that a meter id is returned."""
table_name = AZURE_REPORT_TABLE_MAP["meter"]
meter_id = self.processor._create_meter(self.row, self.accessor)
self.assertIsNotNone(meter_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by("-id").first().id
self.assertEqual(meter_id, id_in_db)
def test_azure_create_cost_entry_line_item(self):
"""Test that Azure line item data is returned properly."""
bill_id = self.processor._create_cost_entry_bill(
self.row, self.accessor)
product_id = self.processor._create_cost_entry_product(
self.row, self.accessor)
meter_id = self.processor._create_meter(self.row, self.accessor)
self.processor._create_cost_entry_line_item(self.row, bill_id,
product_id, meter_id,
self.accessor)
line_item = None
if self.processor.processed_report.line_items:
line_item = self.processor.processed_report.line_items[-1]
self.assertIsNotNone(line_item)
self.assertIn("tags", line_item)
self.assertEqual(line_item.get("cost_entry_bill_id"), bill_id)
self.assertEqual(line_item.get("cost_entry_product_id"), product_id)
self.assertEqual(line_item.get("meter_id"), meter_id)
self.assertIsNotNone(self.processor.line_item_columns)
def test_should_process_row_within_cuttoff_date(self):
"""Test that we correctly determine a row should be processed."""
today = self.date_accessor.today_with_timezone("UTC")
row = {"usagedatetime": today.isoformat()}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
should_process = processor._should_process_row(row, "usagedatetime",
False)
self.assertTrue(should_process)
def test_should_process_row_outside_cuttoff_date(self):
"""Test that we correctly determine a row should be processed."""
today = self.date_accessor.today_with_timezone("UTC")
usage_start = today - relativedelta(days=10)
row = {"usagedatetime": usage_start.isoformat()}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
should_process = processor._should_process_row(row, "usagedatetime",
False)
self.assertFalse(should_process)
def test_get_date_column_filter(self):
"""Test that the Azure specific filter is returned."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
date_filter = processor.get_date_column_filter()
self.assertIn("usage_date__gte", date_filter)
def test_normalize_header(self):
"""Test that headers are always converted to English."""
english_header = (
"DepartmentName,AccountName,AccountOwnerId,SubscriptionGuid,"
"SubscriptionName,ResourceGroup,ResourceLocation,UsageDateTime,"
"ProductName,MeterCategory,MeterSubcategory,MeterId,MeterName,"
"MeterRegion,UnitOfMeasure,UsageQuantity,ResourceRate,PreTaxCost,"
"CostCenter,ConsumedService,ResourceType,InstanceId,Tags,OfferId,"
"AdditionalInfo,ServiceInfo1,ServiceInfo2")
spanish_header = (
"Nombre de departamento (DepartmentName),Nombre de cuenta (AccountName),"
"Id. del propietario de la cuenta (AccountOwnerId),Ubicación de los recursos (SubscriptionGuid),"
"Nombre de la suscripción (SubscriptionName),ResourceGroup (ResourceGroup),"
"Ubicación de los recursos (ResourceLocation),UsageDateTime (UsageDateTime),"
"ProductName (ProductName),Categoría del medidor (MeterCategory),"
"MeterSubcategory (MeterSubcategory),Id. de medidor (MeterId),"
"Nombre del medidor (MeterName),Región del medidor (MeterRegion),"
"Unidad de medida (UnitOfMeasure),UsageQuantity (UsageQuantity),"
"Tasa de recursos (ResourceRate),PreTaxCost (PreTaxCost),"
"Centro de coste (CostCenter),Servicio consumido (ConsumedService),"
"ResourceType (ResourceType),InstanceId (InstanceId),Etiquetas (Tags),"
"OfferId (OfferId),Información adicional (AdditionalInfo),"
"Información del servicio 1 (ServiceInfo1),"
"Información del servicio 2 (ServiceInfo2)")
expected_header = [
"departmentname",
"accountname",
"accountownerid",
"subscriptionguid",
"subscriptionname",
"resourcegroup",
"resourcelocation",
"usagedatetime",
"productname",
"metercategory",
"metersubcategory",
"meterid",
"metername",
"meterregion",
"unitofmeasure",
"usagequantity",
"resourcerate",
"pretaxcost",
"costcenter",
"consumedservice",
"resourcetype",
"instanceid",
"tags",
"offerid",
"additionalinfo",
"serviceinfo1",
"serviceinfo2",
]
self.assertEqual(normalize_header(english_header), expected_header)
self.assertEqual(normalize_header(spanish_header), expected_header)
def test_process_tags(self):
"""Test that both tag string formats are parsed properly."""
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
tag_str_as_json = '{"project":"p1","cost":"management"}'
tag_str_no_json = '"project": "p1","cost": "management"'
expected_tag_str = '{"project":"p1","cost":"management"}'
self.assertEqual(processor._process_tags(tag_str_as_json),
expected_tag_str)
self.assertEqual(processor._process_tags(tag_str_no_json),
expected_tag_str)
self.assertEqual(processor._process_tags(""), "{}")
class AzureReportProcessorTest(MasuTestCase):
"""Test Cases for the AzureReportProcessor object."""
@classmethod
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
_report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP)
_report_tables.pop('line_item_daily_summary', None)
_report_tables.pop('tags_summary', None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r', encoding='utf-8-sig') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def setUp(self):
"""Set up each test."""
super().setUp()
self.assembly_id = '1234'
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider_id,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 1,
'provider_id': self.azure_provider.id,
}
self.accessor = AzureReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
def test_azure_initializer(self):
"""Test Azure initializer."""
self.assertIsNotNone(self.processor._schema_name)
self.assertIsNotNone(self.processor._report_path)
self.assertIsNotNone(self.processor._report_name)
self.assertIsNotNone(self.processor._compression)
self.assertEqual(self.processor._datetime_format,
Config.AZURE_DATETIME_STR_FORMAT)
self.assertEqual(self.processor._batch_size,
Config.REPORT_PROCESSING_BATCH_SIZE)
def test_azure_process(self):
"""Test the processing of an uncompressed Azure file."""
counts = {}
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs('masu.processor.azure.azure_report_processor',
level='INFO') as logger:
self.processor.process()
self.assertIn('INFO:masu.processor.azure.azure_report_processor',
logger.output[0])
self.assertIn(
'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv',
logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
def test_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1):
# Re-init processor so that REPORT_PROCESSING_BATCH_SIZE is 1.
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider_id,
)
# Re-run test with new configuration and verify it's still successful.
self.test_azure_process()
def notest_process_azure_small_batches(self):
"""Test the processing of an uncompressed azure file in small batches."""
with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 1):
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider_id,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
logging.disable(
logging.NOTSET
) # We are currently disabling all logging below CRITICAL in masu/__init__.py
with self.assertLogs('masu.processor.azure.azure_report_processor',
level='INFO') as logger:
processor.process()
self.assertIn(
'INFO:masu.processor.azure.azure_report_processor',
logger.output[0])
self.assertIn(
'costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv',
logger.output[0])
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count > counts[table_name])
def test_azure_process_duplicates(self):
"""Test that row duplicates are not inserted into the DB."""
counts = {}
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider.id,
)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
# Wipe stale data
with schema_context(self.schema):
self.accessor._get_db_obj_query(
AZURE_REPORT_TABLE_MAP['line_item']).delete()
processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider.id,
)
# Process for the second time
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count == counts[table_name])
def test_azure_create_cost_entry_bill(self):
"""Test that a cost entry bill id is returned."""
table_name = AZURE_REPORT_TABLE_MAP['bill']
table = getattr(self.report_schema, table_name)
bill_id = self.processor._create_cost_entry_bill(
self.row, self.accessor)
self.assertIsNotNone(bill_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by('-id').first().id
provider_id = query.order_by('-id').first().provider_id
self.assertEqual(bill_id, id_in_db)
self.assertIsNotNone(provider_id)
def test_azure_create_product(self):
"""Test that a product id is returned."""
table_name = AZURE_REPORT_TABLE_MAP['product']
table = getattr(self.report_schema, table_name)
product_id = self.processor._create_cost_entry_product(
self.row, self.accessor)
self.assertIsNotNone(product_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by('-id').first().id
self.assertEqual(product_id, id_in_db)
def test_azure_create_meter(self):
"""Test that a meter id is returned."""
table_name = AZURE_REPORT_TABLE_MAP['meter']
table = getattr(self.report_schema, table_name)
meter_id = self.processor._create_meter(self.row, self.accessor)
self.assertIsNotNone(meter_id)
query = self.accessor._get_db_obj_query(table_name)
id_in_db = query.order_by('-id').first().id
self.assertEqual(meter_id, id_in_db)
def test_azure_create_cost_entry_line_item(self):
"""Test that Azure line item data is returned properly."""
bill_id = self.processor._create_cost_entry_bill(
self.row, self.accessor)
product_id = self.processor._create_cost_entry_product(
self.row, self.accessor)
meter_id = self.processor._create_meter(self.row, self.accessor)
self.accessor.commit()
self.processor._create_cost_entry_line_item(
self.row,
bill_id,
product_id,
meter_id,
self.accessor,
)
line_item = None
if self.processor.processed_report.line_items:
line_item = self.processor.processed_report.line_items[-1]
self.assertIsNotNone(line_item)
self.assertIn('tags', line_item)
self.assertEqual(line_item.get('cost_entry_bill_id'), bill_id)
self.assertEqual(line_item.get('cost_entry_product_id'), product_id)
self.assertEqual(line_item.get('meter_id'), meter_id)
self.assertIsNotNone(self.processor.line_item_columns)
def test_azure_remove_temp_cur_files(self):
"""Test to remove temporary cost usage files."""
cur_dir = tempfile.mkdtemp()
manifest_data = {"assemblyId": "31727a10-f4b4-43a2-80e5-bef1aaeabfc1"}
manifest = '{}/{}'.format(cur_dir, 'Manifest.json')
with open(manifest, 'w') as outfile:
json.dump(manifest_data, outfile)
file_list = [
{
'file': 'costreport_31727a10-f4b4-43a2-80e5-bef1aaeabfc1.csv',
'processed_date': datetime.datetime(year=2018, month=5, day=3),
},
{
'file': 'costreport_31727a10-f4b4-43a2-80e5-bef1aaeabfc1.csv',
'processed_date': datetime.datetime(year=2018, month=5, day=3),
},
{
'file': 'costreport_2aeb9169-2526-441c-9eca-d7ed015d52bd.csv',
'processed_date': datetime.datetime(year=2018, month=5, day=2),
},
{
'file': 'costreport_6c8487e8-c590-4e6a-b2c2-91a2375c0bad.csv',
'processed_date': datetime.datetime(year=2018, month=5, day=1),
},
{
'file': 'costreport_6c8487e8-c590-4e6a-b2c2-91a2375d0bed.csv',
'processed_date': None,
},
]
expected_delete_list = []
for item in file_list:
path = '{}/{}'.format(cur_dir, item['file'])
f = open(path, 'w')
obj = self.manifest_accessor.get_manifest(self.assembly_id,
self.azure_provider.id)
with ReportStatsDBAccessor(item['file'], obj.id) as stats:
stats.update(last_completed_datetime=item['processed_date'])
f.close()
if (not manifest_data.get('assemblyId') in item['file']):
expected_delete_list.append(path)
removed_files = self.processor.remove_temp_cur_files(cur_dir)
self.assertEqual(sorted(removed_files), sorted(expected_delete_list))
shutil.rmtree(cur_dir)
