def run_test_set(encoder, testRef, testSet, quality, blockSizes, testRuns,
keepOutput):
"""
Execute all tests in the test set.
Args:
encoder (EncoderBase): The encoder to use.
testRef (ResultSet): The test reference results.
testSet (TestSet): The test set.
quality (str): The quality level to execute the test against.
blockSizes (list(str)): The block sizes to execute each test against.
testRuns (int): The number of test repeats to run for each image test.
keepOutput (bool): Should the test preserve output images? This is
only a hint and discarding output may be ignored if the encoder
version used can't do it natively.
Returns:
ResultSet: The test results.
"""
resultSet = trs.ResultSet(testSet.name)
curCount = 0
maxCount = count_test_set(testSet, blockSizes)
dat = (testSet.name, encoder.name, quality)
title = "Test Set: %s / Encoder: %s -%s" % dat
print(title)
print("=" * len(title))
for blkSz in blockSizes:
for image in testSet.tests:
# 3D block sizes require 3D images
if is_3d(blkSz) != image.is3D:
continue
curCount += 1
dat = (curCount, maxCount, blkSz, image.testFile)
print("Running %u/%u %s %s ... " % dat, end='', flush=True)
res = encoder.run_test(image, blkSz, "-%s" % quality, testRuns,
keepOutput)
res = trs.Record(blkSz, image.testFile, res[0], res[1], res[2],
res[3])
resultSet.add_record(res)
if testRef:
refResult = testRef.get_matching_record(res)
res.set_status(determine_result(image, refResult, res))
res.tTimeRel = refResult.tTime / res.tTime
res.cTimeRel = refResult.cTime / res.cTime
res.psnrRel = res.psnr - refResult.psnr
res = format_result(image, refResult, res)
else:
res = format_solo_result(image, res)
print("\r[%3u] %s" % (curCount, res))
return resultSet
def main():
"""
The main function.
Returns:
int: The process return code.
"""
# Parse command lines
args = parse_command_line()
testSetCount = 0
worstResult = trs.Result.NOTRUN
for quality in args.testQual:
for imageSet in args.testSets:
for encoderName in args.encoders:
(encoder, name, outDir, refName) = \
get_encoder_params(encoderName, args.reference, imageSet)
testDir = "Test/Images/%s" % imageSet
testRes = "%s/astc_%s_%s_results.csv" % (outDir, name, quality)
testRef = None
if refName:
dat = (testDir, refName, quality)
testRefPath = "%s/astc_%s_%s_results.csv" % dat
testRef = trs.ResultSet(imageSet)
testRef.load_from_file(testRefPath)
testSetCount += 1
testSet = tts.TestSet(imageSet, testDir, args.profiles,
args.formats, args.testImage)
# The fast and fastest presets are now sufficiently fast that
# the results are noisy without more repeats
testRepeats = args.testRepeats
if quality == "fast" and testRepeats > 1:
testRepeats *= 2
elif quality == "fastest" and testRepeats > 1:
testRepeats *= 4
resultSet = run_test_set(encoder, testRef, testSet, quality,
args.blockSizes, testRepeats,
args.keepOutput)
resultSet.save_to_file(testRes)
if refName:
summary = resultSet.get_results_summary()
worstResult = max(summary.get_worst_result(), worstResult)
print(summary)
if (testSetCount > 1) and (worstResult != trs.Result.NOTRUN):
print("OVERALL STATUS: %s" % worstResult.name)
if worstResult == trs.Result.FAIL:
return 1
return 0
def main():
"""
The main function.
Returns:
int: The process return code.
"""
# Parse command lines
args = parse_command_line()
testSetCount = 0
worstResult = trs.Result.NOTRUN
for imageSet in args.testSets:
for encoderName in args.encoders:
(encoder, name, outDir, refName) = \
get_encoder_params(encoderName, imageSet)
testDir = "Test/Images/%s" % imageSet
testRes = "%s/astc_%s_results.csv" % (outDir, name)
testRef = None
if refName:
testRefPath = "%s/astc_%s_results.csv" % (testDir, refName)
testRef = trs.ResultSet(imageSet)
testRef.load_from_file(testRefPath)
testSetCount += 1
testSet = tts.TestSet(imageSet, testDir, args.profiles,
args.formats)
resultSet = run_test_set(encoder, testRef, testSet,
args.blockSizes, args.testRepeats)
resultSet.save_to_file(testRes)
if refName:
summary = resultSet.get_results_summary()
worstResult = max(summary.get_worst_result(), worstResult)
print(summary)
if (testSetCount > 1) and (worstResult != trs.Result.NOTRUN):
print("OVERALL STATUS: %s" % worstResult.name)
if worstResult == trs.Result.FAIL:
return 1
return 0
def run_test_set(encoder, testRef, testSet, blockSizes, testRuns):
"""
Execute all tests in the test set.
Args:
encoder (EncoderBase): The encoder to use.
testRef (ResultSet): The test reference results.
testSet (TestSet): The test set.
blockSizes (list(str)): The block sizes to execute each test against.
testRuns (int): The number of test repeats to run for each image test.
Returns:
ResultSet: The test results.
"""
resultSet = trs.ResultSet(testSet.name)
curCount = 0
maxCount = count_test_set(testSet, blockSizes)
title = "Test Set: %s / Encoder: %s" % (testSet.name, encoder.name)
print(title)
print("=" * len(title))
for blkSz in blockSizes:
for image in testSet.tests:
# 3D block sizes require 3D images
if is_3d(blkSz) != image.is3D:
continue
curCount += 1
dat = (curCount, maxCount, blkSz, image.testFile)
print("Running %u/%u %s %s ... " % dat, end='', flush=True)
res = encoder.run_test(image, blkSz, "-thorough", testRuns)
res = trs.Record(blkSz, image.testFile, res[0], res[1], res[2])
resultSet.add_record(res)
if testRef:
refResult = testRef.get_matching_record(res)
res.set_status(determine_result(image, refResult, res))
res = format_result(image, refResult, res)
else:
res = format_solo_result(image, res)
print("\r[%3u] %s" % (curCount, res))
return resultSet
def find_reference_results():
"""
Scrape the Test/Images directory for result CSV files and return an
mapping of the result sets.
Returns:
Returns a three deep tree of dictionaries, with the final dict
pointing at a `ResultSet` object. The hierarchy is:
imageSet => quality => encoder => result
"""
scriptDir = os.path.dirname(__file__)
imageDir = os.path.join(scriptDir, "Images")
# Pattern for extracting useful data from the CSV file name
filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv")
# Build a three level dictionary we can write into
results = ddict(lambda: ddict(lambda: ddict()))
# Final all CSVs, load them and store them in the dict tree
for root, dirs, files in os.walk(imageDir):
for name in files:
match = filePat.match(name)
if match:
fullPath = os.path.join(root, name)
encoder = match.group(1)
quality = match.group(2)
imageSet = os.path.basename(root)
if imageSet == "Small":
continue
if imageSet == "Frymire":
continue
testRef = trs.ResultSet(imageSet)
testRef.load_from_file(fullPath)
results[imageSet][quality]["ref-%s" % encoder] = testRef
return results
def find_reference_results():
"""
Scrape the Test/Images directory for result CSV files and return an
mapping of the result sets.
Returns:
Returns a three deep tree of dictionaries, with the final dict
pointing at a `ResultSet` object. The hierarchy is:
imageSet => quality => encoder => result
"""
scriptDir = os.path.dirname(__file__)
imageDir = os.path.join(scriptDir, "Images")
# Pattern for extracting useful data from the CSV file name
filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv")
# Build a three level dictionary we can write into
results = ddict(lambda: ddict(lambda: ddict()))
# Final all CSVs, load them and store them in the dict tree
for root, dirs, files in os.walk(imageDir):
for name in files:
match = filePat.match(name)
if match:
# Skip results set in the filter
skip = [1 for filt in CONFIG_FILTER if filt.match(name)]
if skip:
continue
fullPath = os.path.join(root, name)
encoder = match.group(1)
quality = match.group(2)
imageSet = os.path.basename(root)
# Skip results set in the filter
skip = [1 for filt in TESTSET_FILTER if filt.match(imageSet)]
if skip:
continue
# Skip results set in the filter
skip = [1 for filt in QUALITY_FILTER if filt.match(quality)]
if skip:
continue
testRef = trs.ResultSet(imageSet)
testRef.load_from_file(fullPath)
patchedRef = trs.ResultSet(imageSet)
for result in testRef.records:
skip = [
1 for filt in BLOCKSIZE_FILTER
if filt.match(result.blkSz)
]
if not skip:
patchedRef.add_record(result)
results[imageSet][quality]["ref-%s" % encoder] = patchedRef
return results
