def test_duplicate_header_fails(self):
with self.assertRaises(DuplicateHeaderException):
math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir,
'eplusout_duplicate_header.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
def test_bad_numeric_raises_exception(self):
"""This tests the ability to handle bad numerics which cause weird errors in MathDiff"""
with self.assertRaises(KeyError):
math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_bad_numeric.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
def test_data_with_holes(self):
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_with_data_holes.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertIn('Big Diffs', response[0]) # diff status
self.assertEqual(24, response[1]) # num records compared
self.assertEqual(1, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_invalid_file_2(self):
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_DOESNOTEXIST.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertIn('unable to open file', response[0]) # diff status
self.assertEqual(0, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_totally_empty_file_2(self):
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_totally_empty.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertIn('empty', response[0]) # diff status
self.assertEqual(0, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_big_diff_in_temp_files(self):
"""This tests the ability to capture diffs in a temperature variable - where relative threshold isn't used"""
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_big_temp_diffs.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertEqual('Big Diffs', response[0]) # diff status
self.assertEqual(24, response[1]) # num records compared
self.assertEqual(1, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_big_diff_in_watts_files(self):
"""This tests the ability to capture diffs in a regular (not-temperature) variable"""
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_big_watt_diffs.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertEqual('Big Diffs', response[0]) # diff status
self.assertEqual(24, response[1]) # num records compared
self.assertEqual(2, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_data_with_extra_column_in_case_2(self):
"""If file 2 has extra columns, the comparison should still work but extra outputs will be ignored"""
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir, 'eplusout_extra_column.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertIn('All Equal', response[0]) # diff status
self.assertEqual(24, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_changed_column_order_equal(self):
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir,
'eplusout_change_column_order.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertEqual('All Equal', response[0]) # diff status
self.assertEqual(24, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_data_with_totally_different_headers(self):
"""Two files that don't have _any_ common headers shouldn't work"""
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir,
'eplusout_totally_different_headers.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertIn('No common fields', response[0]) # diff status
self.assertEqual(0, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_changed_timestamps(self):
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir,
'eplusout_changed_timestamps.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertEqual('Time series do not match',
response[0]) # diff status
self.assertEqual(0, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def test_data_with_more_data_than_headers(self):
"""I don't know how we could get to this situation, but anyway, this tests to ensure that if a file has
more data columns than header columns, the data after the last header column is ignored. A diff is encountered
in the extra column, but it should be ignored."""
response = math_diff(
self.thresh_dict,
os.path.join(self.diff_files_dir, 'eplusout.csv'),
os.path.join(self.diff_files_dir,
'eplusout_more_data_than_headers.csv'),
os.path.join(self.temp_output_dir, 'abs_diff.csv'),
os.path.join(self.temp_output_dir, 'rel_diff.csv'),
os.path.join(self.temp_output_dir, 'math_diff.log'),
os.path.join(self.temp_output_dir, 'summary.csv'),
)
self.assertIn('All Equal', response[0]) # diff status
self.assertEqual(24, response[1]) # num records compared
self.assertEqual(0, response[2]) # big diffs
self.assertEqual(0, response[3]) # small diffs
def process_diffs_for_one_case(self, this_entry, ci_mode=False):
if ci_mode: # in "ci_mode" the build directory is actually the output directory of each file
case_result_dir_1 = self.build_tree_a['build_dir']
case_result_dir_2 = self.build_tree_b['build_dir']
else:
case_result_dir_1 = os.path.join(self.build_tree_a['build_dir'],
self.test_output_dir,
this_entry.basename)
case_result_dir_2 = os.path.join(self.build_tree_b['build_dir'],
self.test_output_dir,
this_entry.basename)
out_dir = case_result_dir_1
# we aren't using math_diff and table_diffs summary csv files, so use blanks
path_to_math_diff_log = ""
path_to_table_diff_log = ""
# shortcut
join = os.path.join
# process the end files first
status_case1 = EndErrSummary.STATUS_MISSING
status_case2 = EndErrSummary.STATUS_MISSING
runtime_case1 = 0
runtime_case2 = 0
end_path = join(case_result_dir_1, 'eplusout.end')
if os.path.exists(end_path):
[status_case1, runtime_case1] = self.process_end_file(end_path)
end_path = join(case_result_dir_2, 'eplusout.end')
if os.path.exists(end_path):
[status_case2, runtime_case2] = self.process_end_file(end_path)
# one quick check here for expect-fatal tests
if this_entry.basename == 'EMSTestMathAndKill':
if status_case1 == EndErrSummary.STATUS_FATAL and status_case2 == EndErrSummary.STATUS_FATAL:
# this is actually what we expect, so add a success result, print a message, and get out
this_entry.add_summary_result(
EndErrSummary(EndErrSummary.STATUS_SUCCESS, runtime_case1,
EndErrSummary.STATUS_SUCCESS, runtime_case2))
self.my_print(
"EMSTestMathAndKill Fatal-ed as expected, continuing with no diff checking on it"
)
return this_entry
# add the initial end/err summary to the entry
this_entry.add_summary_result(
EndErrSummary(status_case1, runtime_case1, status_case2,
runtime_case2))
# Handle the results of the end file before doing anything with diffs
# Case 1: Both end files existed, so E+ did complete
if not any(x == EndErrSummary.STATUS_MISSING
for x in [status_case1, status_case2]):
# Case 1a: Both files are successful
if sum(x == EndErrSummary.STATUS_SUCCESS
for x in [status_case1, status_case2]) == 2:
# Just continue to process diffs
self.my_print("Processing (Diffs) : %s" % this_entry.basename)
# Case 1b: Both completed, but both failed: report that it failed in both cases and return early
elif sum(x == EndErrSummary.STATUS_SUCCESS
for x in [status_case1, status_case2]) == 0:
self.my_print(
"Skipping entry because it has a fatal error in both base and mod cases: %s"
% this_entry.basename)
return this_entry
# Case 1c: Both completed, but one failed: report that it failed in one case and return early
elif sum(x == EndErrSummary.STATUS_SUCCESS
for x in [status_case1, status_case2]) == 1:
self.my_print(
"Skipping an entry because it appears to have a fatal error in one case: %s"
% this_entry.basename)
return this_entry
# Case 2: Both end files DID NOT exist
elif all(x == EndErrSummary.STATUS_MISSING
for x in [status_case1, status_case2]):
self.my_print(
"Skipping entry because it failed (crashed) in both base and mod cases: %s"
% this_entry.basename)
return this_entry
# Case 3: Both end files DID NOT exist
elif sum(x == EndErrSummary.STATUS_MISSING
for x in [status_case1, status_case2]) == 1:
self.my_print(
"Skipping an entry because it appears to have failed (crashed) in one case: %s"
% this_entry.basename)
return this_entry
# Case 4: Unhandled combination
else: # pragma: no cover -- I don't think we can get here
self.my_print(
"Skipping an entry because it has an unknown end status: %s" %
this_entry.basename)
return this_entry
# Load diffing threshold dictionary
thresh_dict = td.ThreshDict(self.thresh_dict_file)
# Do Math (CSV) Diffs
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.csv'):
this_entry.add_math_differences(
MathDifferences(
math_diff.math_diff(
thresh_dict, join(case_result_dir_1, 'eplusout.csv'),
join(case_result_dir_2, 'eplusout.csv'),
join(out_dir, 'eplusout.csv.absdiff.csv'),
join(out_dir, 'eplusout.csv.percdiff.csv'),
join(out_dir, 'eplusout.csv.diffsummary.csv'),
path_to_math_diff_log)), MathDifferences.ESO)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusmtr.csv'):
this_entry.add_math_differences(
MathDifferences(
math_diff.math_diff(
thresh_dict, join(case_result_dir_1, 'eplusmtr.csv'),
join(case_result_dir_2, 'eplusmtr.csv'),
join(out_dir, 'eplusmtr.csv.absdiff.csv'),
join(out_dir, 'eplusmtr.csv.percdiff.csv'),
join(out_dir, 'eplusmtr.csv.diffsummary.csv'),
path_to_math_diff_log)), MathDifferences.MTR)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'epluszsz.csv'):
this_entry.add_math_differences(
MathDifferences(
math_diff.math_diff(
thresh_dict, join(case_result_dir_1, 'epluszsz.csv'),
join(case_result_dir_2, 'epluszsz.csv'),
join(out_dir, 'epluszsz.csv.absdiff.csv'),
join(out_dir, 'epluszsz.csv.percdiff.csv'),
join(out_dir, 'epluszsz.csv.diffsummary.csv'),
path_to_math_diff_log)), MathDifferences.ZSZ)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusssz.csv'):
this_entry.add_math_differences(
MathDifferences(
math_diff.math_diff(
thresh_dict, join(case_result_dir_1, 'eplusssz.csv'),
join(case_result_dir_2, 'eplusssz.csv'),
join(out_dir, 'eplusssz.csv.absdiff.csv'),
join(out_dir, 'eplusssz.csv.percdiff.csv'),
join(out_dir, 'eplusssz.csv.diffsummary.csv'),
path_to_math_diff_log)), MathDifferences.SSZ)
# Do Tabular (HTML) Diffs
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplustbl.htm'):
this_entry.add_table_differences(
TableDifferences(
table_diff.table_diff(
thresh_dict, join(case_result_dir_1, 'eplustbl.htm'),
join(case_result_dir_2, 'eplustbl.htm'),
join(out_dir, 'eplustbl.htm.absdiff.htm'),
join(out_dir, 'eplustbl.htm.percdiff.htm'),
join(out_dir, 'eplustbl.htm.summarydiff.htm'),
path_to_table_diff_log)))
# Do Textual Diffs
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.audit'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.audit'),
join(case_result_dir_2, 'eplusout.audit'),
join(out_dir, 'eplusout.audit.diff'))),
TextDifferences.AUD)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.bnd'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.bnd'),
join(case_result_dir_2, 'eplusout.bnd'),
join(out_dir, 'eplusout.bnd.diff'))),
TextDifferences.BND)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.dxf'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.dxf'),
join(case_result_dir_2, 'eplusout.dxf'),
join(out_dir, 'eplusout.dxf.diff'))),
TextDifferences.DXF)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.eio'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.eio'),
join(case_result_dir_2, 'eplusout.eio'),
join(out_dir, 'eplusout.eio.diff'))),
TextDifferences.EIO)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.mdd'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.mdd'),
join(case_result_dir_2, 'eplusout.mdd'),
join(out_dir, 'eplusout.mdd.diff'))),
TextDifferences.MDD)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.mtd'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.mtd'),
join(case_result_dir_2, 'eplusout.mtd'),
join(out_dir, 'eplusout.mtd.diff'))),
TextDifferences.MTD)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.rdd'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.rdd'),
join(case_result_dir_2, 'eplusout.rdd'),
join(out_dir, 'eplusout.rdd.diff'))),
TextDifferences.RDD)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.shd'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.shd'),
join(case_result_dir_2, 'eplusout.shd'),
join(out_dir, 'eplusout.shd.diff'))),
TextDifferences.SHD)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.err'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.err'),
join(case_result_dir_2, 'eplusout.err'),
join(out_dir, 'eplusout.err.diff'))),
TextDifferences.ERR)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.delightin'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.delightin'),
join(case_result_dir_2, 'eplusout.delightin'),
join(out_dir, 'eplusout.delightin.diff'))),
TextDifferences.DL_IN)
if self.both_files_exist(case_result_dir_1, case_result_dir_2,
'eplusout.delightout'):
this_entry.add_text_differences(
TextDifferences(
self.diff_text_files(
join(case_result_dir_1, 'eplusout.delightout'),
join(case_result_dir_2, 'eplusout.delightout'),
join(out_dir, 'eplusout.delightout.diff'))),
TextDifferences.DL_OUT)
# return the updated entry
return this_entry
