def run_test_set(encoder, testRef, testSet, quality, blockSizes, testRuns, keepOutput): """ Execute all tests in the test set. Args: encoder (EncoderBase): The encoder to use. testRef (ResultSet): The test reference results. testSet (TestSet): The test set. quality (str): The quality level to execute the test against. blockSizes (list(str)): The block sizes to execute each test against. testRuns (int): The number of test repeats to run for each image test. keepOutput (bool): Should the test preserve output images? This is only a hint and discarding output may be ignored if the encoder version used can't do it natively. Returns: ResultSet: The test results. """ resultSet = trs.ResultSet(testSet.name) curCount = 0 maxCount = count_test_set(testSet, blockSizes) dat = (testSet.name, encoder.name, quality) title = "Test Set: %s / Encoder: %s -%s" % dat print(title) print("=" * len(title)) for blkSz in blockSizes: for image in testSet.tests: # 3D block sizes require 3D images if is_3d(blkSz) != image.is3D: continue curCount += 1 dat = (curCount, maxCount, blkSz, image.testFile) print("Running %u/%u %s %s ... " % dat, end='', flush=True) res = encoder.run_test(image, blkSz, "-%s" % quality, testRuns, keepOutput) res = trs.Record(blkSz, image.testFile, res[0], res[1], res[2], res[3]) resultSet.add_record(res) if testRef: refResult = testRef.get_matching_record(res) res.set_status(determine_result(image, refResult, res)) res.tTimeRel = refResult.tTime / res.tTime res.cTimeRel = refResult.cTime / res.cTime res.psnrRel = res.psnr - refResult.psnr res = format_result(image, refResult, res) else: res = format_solo_result(image, res) print("\r[%3u] %s" % (curCount, res)) return resultSet
def main(): """ The main function. Returns: int: The process return code. """ # Parse command lines args = parse_command_line() testSetCount = 0 worstResult = trs.Result.NOTRUN for quality in args.testQual: for imageSet in args.testSets: for encoderName in args.encoders: (encoder, name, outDir, refName) = \ get_encoder_params(encoderName, args.reference, imageSet) testDir = "Test/Images/%s" % imageSet testRes = "%s/astc_%s_%s_results.csv" % (outDir, name, quality) testRef = None if refName: dat = (testDir, refName, quality) testRefPath = "%s/astc_%s_%s_results.csv" % dat testRef = trs.ResultSet(imageSet) testRef.load_from_file(testRefPath) testSetCount += 1 testSet = tts.TestSet(imageSet, testDir, args.profiles, args.formats, args.testImage) # The fast and fastest presets are now sufficiently fast that # the results are noisy without more repeats testRepeats = args.testRepeats if quality == "fast" and testRepeats > 1: testRepeats *= 2 elif quality == "fastest" and testRepeats > 1: testRepeats *= 4 resultSet = run_test_set(encoder, testRef, testSet, quality, args.blockSizes, testRepeats, args.keepOutput) resultSet.save_to_file(testRes) if refName: summary = resultSet.get_results_summary() worstResult = max(summary.get_worst_result(), worstResult) print(summary) if (testSetCount > 1) and (worstResult != trs.Result.NOTRUN): print("OVERALL STATUS: %s" % worstResult.name) if worstResult == trs.Result.FAIL: return 1 return 0
def main(): """ The main function. Returns: int: The process return code. """ # Parse command lines args = parse_command_line() testSetCount = 0 worstResult = trs.Result.NOTRUN for imageSet in args.testSets: for encoderName in args.encoders: (encoder, name, outDir, refName) = \ get_encoder_params(encoderName, imageSet) testDir = "Test/Images/%s" % imageSet testRes = "%s/astc_%s_results.csv" % (outDir, name) testRef = None if refName: testRefPath = "%s/astc_%s_results.csv" % (testDir, refName) testRef = trs.ResultSet(imageSet) testRef.load_from_file(testRefPath) testSetCount += 1 testSet = tts.TestSet(imageSet, testDir, args.profiles, args.formats) resultSet = run_test_set(encoder, testRef, testSet, args.blockSizes, args.testRepeats) resultSet.save_to_file(testRes) if refName: summary = resultSet.get_results_summary() worstResult = max(summary.get_worst_result(), worstResult) print(summary) if (testSetCount > 1) and (worstResult != trs.Result.NOTRUN): print("OVERALL STATUS: %s" % worstResult.name) if worstResult == trs.Result.FAIL: return 1 return 0
def run_test_set(encoder, testRef, testSet, blockSizes, testRuns): """ Execute all tests in the test set. Args: encoder (EncoderBase): The encoder to use. testRef (ResultSet): The test reference results. testSet (TestSet): The test set. blockSizes (list(str)): The block sizes to execute each test against. testRuns (int): The number of test repeats to run for each image test. Returns: ResultSet: The test results. """ resultSet = trs.ResultSet(testSet.name) curCount = 0 maxCount = count_test_set(testSet, blockSizes) title = "Test Set: %s / Encoder: %s" % (testSet.name, encoder.name) print(title) print("=" * len(title)) for blkSz in blockSizes: for image in testSet.tests: # 3D block sizes require 3D images if is_3d(blkSz) != image.is3D: continue curCount += 1 dat = (curCount, maxCount, blkSz, image.testFile) print("Running %u/%u %s %s ... " % dat, end='', flush=True) res = encoder.run_test(image, blkSz, "-thorough", testRuns) res = trs.Record(blkSz, image.testFile, res[0], res[1], res[2]) resultSet.add_record(res) if testRef: refResult = testRef.get_matching_record(res) res.set_status(determine_result(image, refResult, res)) res = format_result(image, refResult, res) else: res = format_solo_result(image, res) print("\r[%3u] %s" % (curCount, res)) return resultSet
def find_reference_results(): """ Scrape the Test/Images directory for result CSV files and return an mapping of the result sets. Returns: Returns a three deep tree of dictionaries, with the final dict pointing at a `ResultSet` object. The hierarchy is: imageSet => quality => encoder => result """ scriptDir = os.path.dirname(__file__) imageDir = os.path.join(scriptDir, "Images") # Pattern for extracting useful data from the CSV file name filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv") # Build a three level dictionary we can write into results = ddict(lambda: ddict(lambda: ddict())) # Final all CSVs, load them and store them in the dict tree for root, dirs, files in os.walk(imageDir): for name in files: match = filePat.match(name) if match: fullPath = os.path.join(root, name) encoder = match.group(1) quality = match.group(2) imageSet = os.path.basename(root) if imageSet == "Small": continue if imageSet == "Frymire": continue testRef = trs.ResultSet(imageSet) testRef.load_from_file(fullPath) results[imageSet][quality]["ref-%s" % encoder] = testRef return results
def find_reference_results(): """ Scrape the Test/Images directory for result CSV files and return an mapping of the result sets. Returns: Returns a three deep tree of dictionaries, with the final dict pointing at a `ResultSet` object. The hierarchy is: imageSet => quality => encoder => result """ scriptDir = os.path.dirname(__file__) imageDir = os.path.join(scriptDir, "Images") # Pattern for extracting useful data from the CSV file name filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv") # Build a three level dictionary we can write into results = ddict(lambda: ddict(lambda: ddict())) # Final all CSVs, load them and store them in the dict tree for root, dirs, files in os.walk(imageDir): for name in files: match = filePat.match(name) if match: # Skip results set in the filter skip = [1 for filt in CONFIG_FILTER if filt.match(name)] if skip: continue fullPath = os.path.join(root, name) encoder = match.group(1) quality = match.group(2) imageSet = os.path.basename(root) # Skip results set in the filter skip = [1 for filt in TESTSET_FILTER if filt.match(imageSet)] if skip: continue # Skip results set in the filter skip = [1 for filt in QUALITY_FILTER if filt.match(quality)] if skip: continue testRef = trs.ResultSet(imageSet) testRef.load_from_file(fullPath) patchedRef = trs.ResultSet(imageSet) for result in testRef.records: skip = [ 1 for filt in BLOCKSIZE_FILTER if filt.match(result.blkSz) ] if not skip: patchedRef.add_record(result) results[imageSet][quality]["ref-%s" % encoder] = patchedRef return results