def pkgtest_cmp(args): gnur_filename = args[0] fastr_filename = args[1] if len(args) >= 4: test_output_filters = args[2] pkg_name = args[3] else: test_output_filters = None pkg_name = None dump_preprocessed = args[4] if len(args) >= 5 else None filters = select_filters_for_package(args[2], pkg_name) if len(args) >= 3 else None with open(gnur_filename) as f: gnur_content = f.readlines() with open(fastr_filename) as f: fastr_content = f.readlines() from fuzzy_compare import fuzzy_compare return fuzzy_compare(gnur_content, fastr_content, gnur_filename, fastr_filename, filters, dump_preprocessed)
def _set_test_status(fastr_test_info): def _failed_outputs(outputs): ''' return True iff outputs has any .fail files ''' for _, testfile_status in outputs.iteritems(): if testfile_status.status == "FAILED": return [testfile_status.abspath] return False gnur_test_info = dict() for pkg, _ in fastr_test_info.iteritems(): _get_test_outputs('gnur', pkg, gnur_test_info) # gnur is definitive so drive off that for pkg in gnur_test_info.keys(): logging.info('BEGIN checking ' + pkg) gnur_test_status = gnur_test_info[pkg] fastr_test_status = fastr_test_info[pkg] gnur_outputs = gnur_test_status.testfile_outputs fastr_outputs = fastr_test_status.testfile_outputs gnur_failed_outputs = _failed_outputs(gnur_outputs) if gnur_failed_outputs: # What this likely means is that some native package is not # installed on the system so GNUR can't run the tests. # Ideally this never happens. logging.info("{0}: GnuR test had .fail outputs: {1}".format( pkg, str(gnur_failed_outputs))) fastr_failed_outputs = _failed_outputs(fastr_outputs) if fastr_failed_outputs: # In addition to the similar comment for GNU R, this can happen # if, say, the JVM crashes (possible with native code packages) logging.info("{0}: FastR test had .fail outputs: {1}".format( pkg, str(fastr_failed_outputs))) fastr_test_status.set_status_code("FAILED") # Now for each successful GNU R output we compare content (assuming FastR didn't fail) for gnur_test_output_relpath, gnur_testfile_status in gnur_outputs.iteritems( ): # If FastR does not have a corresponding test output file ... if not gnur_test_output_relpath in fastr_outputs: # FastR crashed on this test fastr_test_status.set_status_code("FAILED") logging.info("{0}: FastR is missing output file: {1}".format( pkg, gnur_test_output_relpath)) continue # Get corresponding FastR test output file fastr_testfile_status = fastr_outputs[gnur_test_output_relpath] # Can't compare if either GNUR or FastR failed if gnur_testfile_status.status == "FAILED": fastr_testfile_status.set_status_code("INDETERMINATE") continue # If the test output file's status is "FAILED" at this point, we know that there was a ".fail" output # file. So, don't do fuzzy-compare. if fastr_testfile_status.status == "FAILED": # It may only be fuzzy-compare because if we would have a test framework, the status would not be # "FAILED" since a test framework cannot produce ".fail" output files. continue gnur_content = None with open(gnur_testfile_status.abspath) as f: gnur_content = f.readlines() fastr_content = None with open(fastr_testfile_status.abspath) as f: fastr_content = f.readlines() # parse custom filters from file filters = select_filters_for_package( os.path.join(_packages_test_project_dir(), "test.output.filter"), pkg) # first, parse file and see if a known test framework has been used detected, ok, skipped, failed = handle_output_file( fastr_testfile_status.abspath, fastr_content) if detected: # If a test framework is used, also parse the summary generated by GnuR to compare numbers. detected, gnur_ok, gnur_skipped, gnur_failed = handle_output_file( gnur_testfile_status.abspath, gnur_content) fastr_invalid_numbers = ok is None or skipped is None and failed is None gnur_invalid_numbers = gnur_ok is None or gnur_skipped is None and gnur_failed is None total_fastr = ok + skipped + failed if not fastr_invalid_numbers else -1 total_gnur = gnur_ok + gnur_skipped + gnur_failed if not gnur_invalid_numbers else -1 if not fastr_invalid_numbers and total_fastr != total_gnur: logging.info( "Different number of tests executed. FastR = {} vs. GnuR = {}" .format(total_fastr, total_gnur)) elif fastr_invalid_numbers: logging.info( "FastR reported invalid numbers of executed tests.") if fastr_invalid_numbers or total_fastr > total_gnur: # If FastR's numbers are invalid or GnuR ran fewer tests than FastR, we cannot trust the FastR numbers fastr_testfile_status.set_report(0, gnur_skipped, gnur_ok + gnur_failed) fastr_test_status.set_status_code("FAILED") fastr_testfile_status.status = "FAILED" elif total_fastr < total_gnur: # If FastR ran fewer tests than GnuR, we complement the missing ones as failing fastr_testfile_status.set_report( ok, skipped, failed + (total_gnur - total_fastr)) fastr_test_status.set_status_code("FAILED") fastr_testfile_status.status = "FAILED" else: # The total numbers are equal, so we are fine. fastr_testfile_status.status = "OK" fastr_testfile_status.set_report(ok, skipped, failed) else: result, n_tests_passed, n_tests_failed = fuzzy_compare( gnur_content, fastr_content, gnur_testfile_status.abspath, fastr_testfile_status.abspath, custom_filters=filters, dump_preprocessed=get_opts().dump_preprocessed) if result == -1: logging.info("{0}: content malformed: {1}".format( pkg, gnur_test_output_relpath)) fastr_test_status.set_status_code("INDETERMINATE") # we don't know how many tests are in there, so consider the whole file to be one big skipped test fastr_testfile_status.set_report(0, 1, 0) elif result != 0: fastr_test_status.set_status_code("FAILED") fastr_testfile_status.status = "FAILED" fastr_testfile_status.set_report(n_tests_passed, 0, n_tests_failed) logging.info("{0}: FastR output mismatch: {1}".format( pkg, gnur_test_output_relpath)) logging.info(" output mismatch file: {0}".format( join(_pkg_testdir('fastr', pkg), gnur_test_output_relpath))) logging.info(" output mismatch file: {0}".format( join(_pkg_testdir('gnur', pkg), gnur_test_output_relpath))) else: fastr_testfile_status.status = "OK" fastr_testfile_status.set_report(n_tests_passed, 0, n_tests_failed) # we started out as UNKNOWN if not (fastr_test_status.status == "INDETERMINATE" or fastr_test_status.status == "FAILED"): fastr_test_status.set_status_code("OK") # write out a file with the test status for each output (that exists) with open(join(_pkg_testdir('fastr', pkg), 'testfile_status'), 'w') as f: f.write( '# <file path> <tests passed> <tests skipped> <tests failed>\n' ) for fastr_relpath, fastr_testfile_status in fastr_outputs.iteritems( ): logging.info( "generating testfile_status for {0}".format(fastr_relpath)) relpath = fastr_relpath test_output_file = join(_pkg_testdir('fastr', pkg), relpath) if os.path.exists(test_output_file): ok, skipped, failed = fastr_testfile_status.get_report() f.write("{0} {1} {2} {3} {4}\n".format( relpath, ok, skipped, failed, fastr_testfile_status.test_time)) elif fastr_testfile_status.status == "FAILED": # In case of status == "FAILED", also try suffix ".fail" because we just do not know if the test # failed and finished or just never finished. relpath_fail = fastr_relpath + ".fail" test_output_file_fail = join(_pkg_testdir('fastr', pkg), relpath_fail) if os.path.exists(test_output_file_fail): ok, skipped, failed = fastr_testfile_status.get_report( ) f.write("{0} {1} {2} {3} {4}\n".format( relpath_fail, ok, skipped, failed, fastr_testfile_status.test_time)) else: logging.info("File {0} or {1} does not exist".format( test_output_file, test_output_file_fail)) else: logging.info( "File {0} does not exist".format(test_output_file)) with open(join(_pkg_testdir('fastr', pkg), 'test_time'), 'w') as f: f.write(str(fastr_test_status.test_time)) logging.info('END checking ' + pkg)