def test_duplicate_header_fails(self): with self.assertRaises(DuplicateHeaderException): math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_duplicate_header.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), )
def test_bad_numeric_raises_exception(self): """This tests the ability to handle bad numerics which cause weird errors in MathDiff""" with self.assertRaises(KeyError): math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_bad_numeric.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), )
def test_data_with_holes(self): response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_with_data_holes.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertIn('Big Diffs', response[0]) # diff status self.assertEqual(24, response[1]) # num records compared self.assertEqual(1, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_invalid_file_2(self): response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_DOESNOTEXIST.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertIn('unable to open file', response[0]) # diff status self.assertEqual(0, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_totally_empty_file_2(self): response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_totally_empty.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertIn('empty', response[0]) # diff status self.assertEqual(0, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_big_diff_in_temp_files(self): """This tests the ability to capture diffs in a temperature variable - where relative threshold isn't used""" response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_big_temp_diffs.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertEqual('Big Diffs', response[0]) # diff status self.assertEqual(24, response[1]) # num records compared self.assertEqual(1, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_big_diff_in_watts_files(self): """This tests the ability to capture diffs in a regular (not-temperature) variable""" response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_big_watt_diffs.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertEqual('Big Diffs', response[0]) # diff status self.assertEqual(24, response[1]) # num records compared self.assertEqual(2, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_data_with_extra_column_in_case_2(self): """If file 2 has extra columns, the comparison should still work but extra outputs will be ignored""" response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_extra_column.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertIn('All Equal', response[0]) # diff status self.assertEqual(24, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_changed_column_order_equal(self): response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_change_column_order.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertEqual('All Equal', response[0]) # diff status self.assertEqual(24, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_data_with_totally_different_headers(self): """Two files that don't have _any_ common headers shouldn't work""" response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_totally_different_headers.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertIn('No common fields', response[0]) # diff status self.assertEqual(0, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_changed_timestamps(self): response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_changed_timestamps.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertEqual('Time series do not match', response[0]) # diff status self.assertEqual(0, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def test_data_with_more_data_than_headers(self): """I don't know how we could get to this situation, but anyway, this tests to ensure that if a file has more data columns than header columns, the data after the last header column is ignored. A diff is encountered in the extra column, but it should be ignored.""" response = math_diff( self.thresh_dict, os.path.join(self.diff_files_dir, 'eplusout.csv'), os.path.join(self.diff_files_dir, 'eplusout_more_data_than_headers.csv'), os.path.join(self.temp_output_dir, 'abs_diff.csv'), os.path.join(self.temp_output_dir, 'rel_diff.csv'), os.path.join(self.temp_output_dir, 'math_diff.log'), os.path.join(self.temp_output_dir, 'summary.csv'), ) self.assertIn('All Equal', response[0]) # diff status self.assertEqual(24, response[1]) # num records compared self.assertEqual(0, response[2]) # big diffs self.assertEqual(0, response[3]) # small diffs
def process_diffs_for_one_case(self, this_entry, ci_mode=False): if ci_mode: # in "ci_mode" the build directory is actually the output directory of each file case_result_dir_1 = self.build_tree_a['build_dir'] case_result_dir_2 = self.build_tree_b['build_dir'] else: case_result_dir_1 = os.path.join(self.build_tree_a['build_dir'], self.test_output_dir, this_entry.basename) case_result_dir_2 = os.path.join(self.build_tree_b['build_dir'], self.test_output_dir, this_entry.basename) out_dir = case_result_dir_1 # we aren't using math_diff and table_diffs summary csv files, so use blanks path_to_math_diff_log = "" path_to_table_diff_log = "" # shortcut join = os.path.join # process the end files first status_case1 = EndErrSummary.STATUS_MISSING status_case2 = EndErrSummary.STATUS_MISSING runtime_case1 = 0 runtime_case2 = 0 end_path = join(case_result_dir_1, 'eplusout.end') if os.path.exists(end_path): [status_case1, runtime_case1] = self.process_end_file(end_path) end_path = join(case_result_dir_2, 'eplusout.end') if os.path.exists(end_path): [status_case2, runtime_case2] = self.process_end_file(end_path) # one quick check here for expect-fatal tests if this_entry.basename == 'EMSTestMathAndKill': if status_case1 == EndErrSummary.STATUS_FATAL and status_case2 == EndErrSummary.STATUS_FATAL: # this is actually what we expect, so add a success result, print a message, and get out this_entry.add_summary_result( EndErrSummary(EndErrSummary.STATUS_SUCCESS, runtime_case1, EndErrSummary.STATUS_SUCCESS, runtime_case2)) self.my_print( "EMSTestMathAndKill Fatal-ed as expected, continuing with no diff checking on it" ) return this_entry # add the initial end/err summary to the entry this_entry.add_summary_result( EndErrSummary(status_case1, runtime_case1, status_case2, runtime_case2)) # Handle the results of the end file before doing anything with diffs # Case 1: Both end files existed, so E+ did complete if not any(x == EndErrSummary.STATUS_MISSING for x in [status_case1, status_case2]): # Case 1a: Both files are successful if sum(x == EndErrSummary.STATUS_SUCCESS for x in [status_case1, status_case2]) == 2: # Just continue to process diffs self.my_print("Processing (Diffs) : %s" % this_entry.basename) # Case 1b: Both completed, but both failed: report that it failed in both cases and return early elif sum(x == EndErrSummary.STATUS_SUCCESS for x in [status_case1, status_case2]) == 0: self.my_print( "Skipping entry because it has a fatal error in both base and mod cases: %s" % this_entry.basename) return this_entry # Case 1c: Both completed, but one failed: report that it failed in one case and return early elif sum(x == EndErrSummary.STATUS_SUCCESS for x in [status_case1, status_case2]) == 1: self.my_print( "Skipping an entry because it appears to have a fatal error in one case: %s" % this_entry.basename) return this_entry # Case 2: Both end files DID NOT exist elif all(x == EndErrSummary.STATUS_MISSING for x in [status_case1, status_case2]): self.my_print( "Skipping entry because it failed (crashed) in both base and mod cases: %s" % this_entry.basename) return this_entry # Case 3: Both end files DID NOT exist elif sum(x == EndErrSummary.STATUS_MISSING for x in [status_case1, status_case2]) == 1: self.my_print( "Skipping an entry because it appears to have failed (crashed) in one case: %s" % this_entry.basename) return this_entry # Case 4: Unhandled combination else: # pragma: no cover -- I don't think we can get here self.my_print( "Skipping an entry because it has an unknown end status: %s" % this_entry.basename) return this_entry # Load diffing threshold dictionary thresh_dict = td.ThreshDict(self.thresh_dict_file) # Do Math (CSV) Diffs if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.csv'): this_entry.add_math_differences( MathDifferences( math_diff.math_diff( thresh_dict, join(case_result_dir_1, 'eplusout.csv'), join(case_result_dir_2, 'eplusout.csv'), join(out_dir, 'eplusout.csv.absdiff.csv'), join(out_dir, 'eplusout.csv.percdiff.csv'), join(out_dir, 'eplusout.csv.diffsummary.csv'), path_to_math_diff_log)), MathDifferences.ESO) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusmtr.csv'): this_entry.add_math_differences( MathDifferences( math_diff.math_diff( thresh_dict, join(case_result_dir_1, 'eplusmtr.csv'), join(case_result_dir_2, 'eplusmtr.csv'), join(out_dir, 'eplusmtr.csv.absdiff.csv'), join(out_dir, 'eplusmtr.csv.percdiff.csv'), join(out_dir, 'eplusmtr.csv.diffsummary.csv'), path_to_math_diff_log)), MathDifferences.MTR) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'epluszsz.csv'): this_entry.add_math_differences( MathDifferences( math_diff.math_diff( thresh_dict, join(case_result_dir_1, 'epluszsz.csv'), join(case_result_dir_2, 'epluszsz.csv'), join(out_dir, 'epluszsz.csv.absdiff.csv'), join(out_dir, 'epluszsz.csv.percdiff.csv'), join(out_dir, 'epluszsz.csv.diffsummary.csv'), path_to_math_diff_log)), MathDifferences.ZSZ) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusssz.csv'): this_entry.add_math_differences( MathDifferences( math_diff.math_diff( thresh_dict, join(case_result_dir_1, 'eplusssz.csv'), join(case_result_dir_2, 'eplusssz.csv'), join(out_dir, 'eplusssz.csv.absdiff.csv'), join(out_dir, 'eplusssz.csv.percdiff.csv'), join(out_dir, 'eplusssz.csv.diffsummary.csv'), path_to_math_diff_log)), MathDifferences.SSZ) # Do Tabular (HTML) Diffs if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplustbl.htm'): this_entry.add_table_differences( TableDifferences( table_diff.table_diff( thresh_dict, join(case_result_dir_1, 'eplustbl.htm'), join(case_result_dir_2, 'eplustbl.htm'), join(out_dir, 'eplustbl.htm.absdiff.htm'), join(out_dir, 'eplustbl.htm.percdiff.htm'), join(out_dir, 'eplustbl.htm.summarydiff.htm'), path_to_table_diff_log))) # Do Textual Diffs if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.audit'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.audit'), join(case_result_dir_2, 'eplusout.audit'), join(out_dir, 'eplusout.audit.diff'))), TextDifferences.AUD) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.bnd'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.bnd'), join(case_result_dir_2, 'eplusout.bnd'), join(out_dir, 'eplusout.bnd.diff'))), TextDifferences.BND) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.dxf'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.dxf'), join(case_result_dir_2, 'eplusout.dxf'), join(out_dir, 'eplusout.dxf.diff'))), TextDifferences.DXF) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.eio'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.eio'), join(case_result_dir_2, 'eplusout.eio'), join(out_dir, 'eplusout.eio.diff'))), TextDifferences.EIO) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.mdd'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.mdd'), join(case_result_dir_2, 'eplusout.mdd'), join(out_dir, 'eplusout.mdd.diff'))), TextDifferences.MDD) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.mtd'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.mtd'), join(case_result_dir_2, 'eplusout.mtd'), join(out_dir, 'eplusout.mtd.diff'))), TextDifferences.MTD) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.rdd'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.rdd'), join(case_result_dir_2, 'eplusout.rdd'), join(out_dir, 'eplusout.rdd.diff'))), TextDifferences.RDD) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.shd'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.shd'), join(case_result_dir_2, 'eplusout.shd'), join(out_dir, 'eplusout.shd.diff'))), TextDifferences.SHD) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.err'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.err'), join(case_result_dir_2, 'eplusout.err'), join(out_dir, 'eplusout.err.diff'))), TextDifferences.ERR) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.delightin'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.delightin'), join(case_result_dir_2, 'eplusout.delightin'), join(out_dir, 'eplusout.delightin.diff'))), TextDifferences.DL_IN) if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.delightout'): this_entry.add_text_differences( TextDifferences( self.diff_text_files( join(case_result_dir_1, 'eplusout.delightout'), join(case_result_dir_2, 'eplusout.delightout'), join(out_dir, 'eplusout.delightout.diff'))), TextDifferences.DL_OUT) # return the updated entry return this_entry