def test_process_default(self):
"""Test the processing of an uncompressed file."""
counts = {}
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
if table_name not in ("reporting_ocpusagelineitem_daily",
"reporting_ocpusagelineitem_daily_summary"):
self.assertTrue(count >= counts[table_name])
self.assertFalse(os.path.exists(self.test_report))
def test_process_default_small_batches(self):
"""Test the processing of an uncompressed file in small batches."""
with patch.object(Config, "REPORT_PROCESSING_BATCH_SIZE", 5):
counts = {}
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
if table_name not in (
"reporting_ocpusagelineitem_daily",
"reporting_ocpusagelineitem_daily_summary"):
self.assertTrue(count >= counts[table_name])
def test_process_duplicate_rows_same_file(self):
"""Test that row duplicates are not inserted into the DB."""
data = []
with open(self.test_report, "r") as f:
reader = csv.DictReader(f)
for row in reader:
data.append(row)
expected_count = len(data)
data.extend(data)
tmp_file = "/tmp/test_process_duplicate_rows_same_file.csv"
field_names = data[0].keys()
with open(tmp_file, "w") as f:
writer = csv.DictWriter(f, fieldnames=field_names)
writer.writeheader()
writer.writerows(data)
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=tmp_file,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
table_name = OCP_REPORT_TABLE_MAP["line_item"]
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertEqual(count, expected_count)
def test_process_storage_duplicates(self):
"""Test that row duplicate storage rows are not inserted into the DB."""
counts = {}
processor = OCPReportProcessor(schema_name='acct10001',
report_path=self.storage_report,
compression=UNCOMPRESSED,
provider_id=1)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
counts[table_name] = count
processor = OCPReportProcessor(schema_name='acct10001',
report_path=self.storage_report,
compression=UNCOMPRESSED,
provider_id=1)
# Process for the second time
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
self.assertTrue(count == counts[table_name])
def test_process_usage_and_storage_default(self):
"""Test the processing of an uncompressed storage and usage files."""
storage_processor = OCPReportProcessor(schema_name='acct10001',
report_path=self.storage_report,
compression=UNCOMPRESSED,
provider_id=1)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP['storage_line_item']
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
storage_before_count = report_db._session.query(table).count()
storage_processor.process()
storage_after_count = report_db._session.query(table).count()
self.assertGreater(storage_after_count, storage_before_count)
processor = OCPReportProcessor(schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP['line_item']
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
before_count = report_db._session.query(table).count()
processor.process()
after_count = report_db._session.query(table).count()
self.assertGreater(after_count, before_count)
def test_process_duplicate_rows_same_file(self):
"""Test that row duplicates are not inserted into the DB."""
data = []
with open(self.test_report, 'r') as f:
reader = csv.DictReader(f)
for row in reader:
data.append(row)
expected_count = len(data)
data.extend(data)
tmp_file = '/tmp/test_process_duplicate_rows_same_file.csv'
field_names = data[0].keys()
with open(tmp_file, 'w') as f:
writer = csv.DictWriter(f, fieldnames=field_names)
writer.writeheader()
writer.writerows(data)
processor = OCPReportProcessor(schema_name='acct10001',
report_path=tmp_file,
compression=UNCOMPRESSED,
provider_id=1)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
table_name = OCP_REPORT_TABLE_MAP['line_item']
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
self.assertEqual(count, expected_count)
def test_process_duplicates(self):
"""Test that row duplicates are not inserted into the DB."""
counts = {}
processor = OCPReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1,
)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
processor = OCPReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1,
)
# Process for the second time
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count == counts[table_name])
def test_process_default_small_batches(self):
"""Test the processing of an uncompressed file in small batches."""
with patch.object(Config, 'REPORT_PROCESSING_BATCH_SIZE', 5):
counts = {}
processor = OCPReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
counts[table_name] = count
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
if table_name not in (
'reporting_ocpusagelineitem_daily',
'reporting_ocpusagelineitem_daily_summary',
):
self.assertTrue(count >= counts[table_name])
def test_process_default(self):
"""Test the processing of an uncompressed file."""
counts = {}
processor = OCPReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1,
)
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
counts[table_name] = count
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
count = report_db._session.query(table).count()
if table_name not in (
'reporting_ocpusagelineitem_daily',
'reporting_ocpusagelineitem_daily_summary',
):
self.assertTrue(count >= counts[table_name])
def test_process_usage_and_storage_default(self):
"""Test the processing of an uncompressed storage and usage files."""
storage_processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.storage_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP["storage_line_item"]
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
with schema_context(self.schema):
storage_before_count = table.objects.count()
storage_processor.process()
with schema_context(self.schema):
storage_after_count = table.objects.count()
self.assertGreater(storage_after_count, storage_before_count)
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP["line_item"]
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
with schema_context(self.schema):
before_count = table.objects.count()
processor.process()
with schema_context(self.schema):
after_count = table.objects.count()
self.assertGreater(after_count, before_count)
def test_process_storage_default(self):
"""Test the processing of an uncompressed storagefile."""
processor = OCPReportProcessor(
schema_name='acct10001',
report_path=self.storage_report,
compression=UNCOMPRESSED,
provider_id=1,
)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP['storage_line_item']
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
with schema_context(self.schema):
before_count = table.objects.count()
processor.process()
with schema_context(self.schema):
after_count = table.objects.count()
self.assertGreater(after_count, before_count)
def test_process_node_label_duplicates(self):
"""Test that row duplicate node label rows are not inserted into the DB."""
counts = {}
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.node_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
# Process for the first time
processor.process()
report_db = self.accessor
report_schema = report_db.report_schema
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
counts[table_name] = count
shutil.copy2(self.node_report_path, self.node_report)
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=self.node_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
# Process for the second time
processor.process()
for table_name in self.report_tables:
table = getattr(report_schema, table_name)
with schema_context(self.schema):
count = table.objects.count()
self.assertTrue(count == counts[table_name])
def test_process_usage_and_storage_with_invalid_data(self):
"""Test that processing succeeds when rows are missing data."""
pod_report = f"{self.temp_dir}/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1-invalid.csv"
storage_report = f"{self.temp_dir}/e6b3701e-1e91-433b-b238-a31e49937558_storage-invalid.csv"
pod_data = []
storage_data = []
with open(self.test_report_path) as f:
reader = csv.DictReader(f)
for row in reader:
row["node"] = None
pod_data.append(row)
header = pod_data[0].keys()
with open(pod_report, "w") as f:
writer = csv.DictWriter(f, fieldnames=header)
writer.writeheader()
writer.writerows(pod_data)
with open(self.storage_report_path) as f:
reader = csv.DictReader(f)
for row in reader:
row["persistentvolume"] = None
storage_data.append(row)
header = storage_data[0].keys()
with open(storage_report, "w") as f:
writer = csv.DictWriter(f, fieldnames=header)
writer.writeheader()
writer.writerows(storage_data)
storage_processor = OCPReportProcessor(
schema_name="acct10001",
report_path=storage_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP["storage_line_item"]
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
with schema_context(self.schema):
storage_before_count = table.objects.count()
storage_processor.process()
with schema_context(self.schema):
storage_after_count = table.objects.count()
self.assertEqual(storage_after_count, storage_before_count)
processor = OCPReportProcessor(
schema_name="acct10001",
report_path=pod_report,
compression=UNCOMPRESSED,
provider_uuid=self.ocp_provider_uuid,
)
report_db = self.accessor
table_name = OCP_REPORT_TABLE_MAP["line_item"]
report_schema = report_db.report_schema
table = getattr(report_schema, table_name)
with schema_context(self.schema):
before_count = table.objects.count()
processor.process()
with schema_context(self.schema):
after_count = table.objects.count()
self.assertEqual(after_count, before_count)
