def generate_refs(env, tests, ref_dir):
'''
Computes references for a set of tests and stores them into ref_dir.
'''
print(f'Reference directory: {ref_dir}')
print(f'Generating references for {len(tests)} tests')
# Remove existing references.
if ref_dir.exists():
shutil.rmtree(ref_dir, ignore_errors=True)
success = True
for test in tests:
print(f' {test.name:<60} : ', end='', flush=True)
start_time = time.time()
result, messages = test.generate_images(ref_dir, env.mogwai_exe)
elapsed_time = time.time() - start_time
if result == Test.Result.FAILED:
success = False
# Print result and messages.
status = Test.COLORED_RESULT_STRING[result]
print(f'{status} ({elapsed_time:.1f} s)')
for message in messages:
print(f' {message}')
status = colored('PASSED', 'green') if success else colored(
'FAILED', 'red')
print(f'\nGenerating references {status}.')
return success
def pull_refs(remote_ref_dir, ref_dir):
'''
Pulls reference images from remote_ref_dir to ref_dir.
'''
print(f'Pulling reference images from {remote_ref_dir} : ', end='', flush=True)
success, log = helpers.mirror_folders(remote_ref_dir, ref_dir)
print(colored('OK', 'green') if success else colored('FAILED', 'red'))
if not success:
print(log)
return success
def run_tests(env, tests, compare_only, ref_dir, result_dir):
'''
Runs a set of tests, stores them into result_dir and compares them to ref_dir.
'''
print(f'Result directory: {result_dir}')
print(f'Reference directory: {ref_dir}')
print(f'Running {len(tests)} tests')
success = True
run_date = datetime.datetime.now()
run_start_time = time.time()
for test in tests:
print(f' {test.name:<60} : ', end='', flush=True)
start_time = time.time()
result, messages = test.run(compare_only, ref_dir, result_dir,
env.mogwai_exe, env.image_compare_exe)
elapsed_time = time.time() - start_time
if result == Test.Result.FAILED:
success = False
# Print result and messages.
status = Test.COLORED_RESULT_STRING[result]
print(f'{status} ({elapsed_time:.1f} s)')
for message in messages:
print(f' {message}')
status = colored('PASSED', 'green') if success else colored(
'FAILED', 'red')
print(f'\nImage tests {status}.')
# Setup report.
report = {
'date': run_date.isoformat(),
'result': 'PASSED' if success else 'FAILED',
'tests': [t.name for t in tests],
'duration': time.time() - run_start_time
}
# Write JSON report.
report_file = result_dir / 'report.json'
with open(report_file, 'w') as f:
json.dump(report, f, indent=4)
return success
def run_unit_tests(env, filter_regex):
'''
Run unit tests by running FalcorTest.
The optional filter_regex is used to select specific tests to run.
'''
args = [env.falcor_test_exe]
if filter_regex:
args += ['-test_filter', filter_regex]
try:
process = helpers.run_with_timeout(args, timeout=600)
success = process.returncode == 0
except Exception as e:
print(f'\n\nError running {env.falcor_test_exe}: {e}')
success = False
status = colored('PASSED', 'green') if success else colored(
'FAILED', 'red')
print(f'\nUnit tests {status}.')
return success
def run_unit_tests(env, filter_regex):
'''
Run unit tests by running FalcorTest.
The optional filter_regex is used to select specific tests to run.
'''
args = [str(env.falcor_test_exe)]
if filter_regex:
args += ['--filter', str(filter_regex)]
p = subprocess.Popen(args)
try:
p.communicate(timeout=600)
except subprocess.TimeoutExpired:
p.kill()
print('\n\nProcess killed due to timeout')
success = p.returncode == 0
status = colored('PASSED', 'green') if success else colored(
'FAILED', 'red')
print(f'\nUnit tests {status}.')
return success
class Test:
'''
Represents a single image test.
'''
class Result(Enum):
PASSED = 1
FAILED = 2
SKIPPED = 3
COLORED_RESULT_STRING = {
Result.PASSED: colored('PASSED', 'green'),
Result.FAILED: colored('FAILED', 'red'),
Result.SKIPPED: colored('SKIPPED', 'yellow')
}
RESULT_STRING = {
Result.PASSED: 'PASSED',
Result.FAILED: 'FAILED',
Result.SKIPPED: 'SKIPPED'
}
def __init__(self, script_file, root_dir):
self.script_file = script_file
# Test directory relative to root directory.
self.test_dir = self.script_file.relative_to(root_dir).with_suffix('')
# Test name derived from test directory.
self.name = str(self.test_dir.as_posix())
# Read script header.
try:
self.header = read_header(script_file)
except Exception as e:
print(e)
sys.exit(1)
# Get skipped tests.
self.skip_message = self.header.get('skipped', None)
self.skipped = self.skip_message != None
# Get tolerance.
self.tolerance = self.header.get('tolerance', config.TOLERANCE)
def __repr__(self):
return f'Test(name={self.name},script_file={self.script_file})'
def collect_images(self, image_dir):
'''
Collect all reference and result images in a directory.
Returns paths relative to the given directory.
'''
files = image_dir.iterdir()
files = map(lambda f: f.relative_to(image_dir), files)
files = filter(
lambda f: not str(f).endswith(config.ERROR_IMAGE_SUFFIX), files)
files = filter(lambda f: f.suffix.lower() in config.IMAGE_EXTENSIONS,
files)
return list(files)
def generate_images(self, output_dir, mogwai_exe):
'''
Run Mogwai to generate a set of images and store them in output_dir.
Returns a tuple containing the result code and a list of messages.
'''
# Bail out if test is skipped.
if self.skipped:
return Test.Result.SKIPPED, [self.skip_message
] if self.skip_message != '' else []
# Determine full output directory.
output_dir = output_dir / self.test_dir
output_dir.mkdir(parents=True, exist_ok=True)
# In order to simplify module imports in test scripts, we run Mogwai with
# a working directory set to the directory the test script resides in.
# The working directory also specifies the python search path.
# Here we determine the working directory and a function to resolve paths
# to be relative to the working directory.
cwd = self.script_file.parent
relative_to_cwd = lambda p: os.path.relpath(p, cwd)
# Write helper script to run test.
generate_file = output_dir / 'generate.py'
with open(generate_file, 'w') as f:
f.write(f'fc.outputDir = r"{output_dir}"\n')
f.write(f'm.script(r"{relative_to_cwd(self.script_file)}")\n')
# Run Mogwai to generate images.
try:
process = helpers.run_with_timeout([
mogwai_exe, '-script',
relative_to_cwd(generate_file), '-logfile',
output_dir / 'log.txt', '-silent'
],
cwd=cwd,
stderr=subprocess.PIPE,
timeout=600)
if process.returncode != 0:
errors = list(
map(lambda l: l.decode('utf-8').rstrip(),
process.stderr.readlines()))
return Test.Result.FAILED, errors + [
f'{mogwai_exe} exited with return code {process.returncode}'
]
except Exception as e:
return Test.Result.FAILED, [f'Error running {mogwai_exe}: {e}']
# Bail out if no images have been generated.
if len(self.collect_images(output_dir)) == 0:
return Test.Result.FAILED, ['Test did not generate any images.']
return Test.Result.PASSED, []
def compare_image(self, ref_file, result_file, error_file, tolerance,
image_compare_exe):
'''
Compare two images using ImageCompare.
Returns a tuple containing the a boolean to indicate success and the measured error.
'''
args = [
str(image_compare_exe), '-m', 'mse', '-t',
str(tolerance),
str(ref_file),
str(result_file)
]
if error_file:
args += ['-e', str(error_file)]
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = process.communicate()[0]
error = float(output.strip())
return process.returncode == 0, error
def compare_images(self, ref_dir, result_dir, image_compare_exe):
'''
Run ImageCompare on a set of images in ref_dir and result_dir.
Checks if error between reference and result image is within a given tolerance.
Returns a tuple containing the result code, a list of messages and a list of image reports.
'''
# Bail out if test is skipped.
if self.skipped:
return Test.Result.SKIPPED, [
self.skip_message
] if self.skip_message != '' else [], []
# Determine full directory paths for references and results.
ref_dir = ref_dir / self.test_dir
result_dir = result_dir / self.test_dir
# Make sure directories exist.
if not ref_dir.exists():
return Test.Result.FAILED, [
f'Reference directory "{ref_dir}" does not exist.'
], []
elif not result_dir.exists():
return Test.Result.FAILED, [
f'Result directory "{result_dir}" does not exist.'
], []
# Collect all reference and result images.
ref_images = self.collect_images(ref_dir)
result_images = self.collect_images(result_dir)
# Bail out if no images have been generated.
if len(result_images) == 0:
return Test.Result.FAILED, ['Test did not generate any images.'
], []
result = Test.Result.PASSED
messages = []
image_reports = []
# Compare every result image with the corresponding reference image and report missing references.
for image in result_images:
if not image in ref_images:
result = Test.Result.FAILED
messages.append(
f'Test has generated image "{image}" with no corresponding reference image.'
)
continue
ref_file = ref_dir / image
result_file = result_dir / image
error_file = result_dir / (str(image) + config.ERROR_IMAGE_SUFFIX)
compare_success, compare_error = self.compare_image(
ref_file, result_file, error_file, self.tolerance,
image_compare_exe)
if not compare_success:
result = Test.Result.FAILED
messages.append(
f'Test image "{image}" failed with error {compare_error}.')
image_reports.append({
'name': str(image),
'success': compare_success,
'error': compare_error,
'tolerance': self.tolerance
})
# Report missing result images for existing reference images.
for image in ref_images:
if not image in result_images:
result = Test.Result.FAILED
messages.append(
f'Test has not generated an image for the corresponding reference image "{image}".'
)
return result, messages, image_reports
def run(self, compare_only, ref_dir, result_dir, mogwai_exe,
image_compare_exe):
'''
Run the image test.
First, result images are generated (unless compare_only is True).
Second, result images are compared against reference images.
Third, writes a JSON report to the result_dir containing details on the test run.
Returns a tuple containing the result code and a list of messages.
'''
# Setup report.
report = {
'name': self.name,
'ref_dir': str(ref_dir / self.test_dir),
'images': []
}
start_time = time.time()
result = Test.Result.PASSED
messages = []
# Generate results images.
if not compare_only:
result, messages = self.generate_images(result_dir, mogwai_exe)
# Compare to references.
if result == Test.Result.PASSED:
result, messages, report['images'] = self.compare_images(
ref_dir, result_dir, image_compare_exe)
# Finish report.
report['result'] = Test.RESULT_STRING[result]
report['messages'] = messages
report['duration'] = time.time() - start_time
# Write JSON report.
report_dir = result_dir / self.test_dir
report_dir.mkdir(parents=True, exist_ok=True)
report_file = report_dir / 'report.json'
with open(report_file, 'w') as f:
json.dump(report, f, indent=4)
return result, messages
def main():
available_configs = ', '.join(config.BUILD_CONFIGS.keys())
parser = argparse.ArgumentParser(
description="Utility for running image tests.")
parser.add_argument('-c',
'--config',
type=str,
action='store',
help=f'Build configuration: {available_configs}',
default=config.DEFAULT_BUILD_CONFIG)
parser.add_argument('-e',
'--environment',
type=str,
action='store',
help='Environment',
default=config.DEFAULT_ENVIRONMENT)
parser.add_argument('-l',
'--list',
action='store_true',
help='List available tests')
parser.add_argument('-f',
'--filter',
type=str,
action='store',
help='Regular expression for filtering tests to run')
parser.add_argument(
'-b',
'--ref-branch',
help='Reference branch to compare against (defaults to master branch)',
default='master')
parser.add_argument(
'--compare-only',
action='store_true',
help=
'Compare previous results against references without generating new images'
)
parser.add_argument(
'--gen-refs',
action='store_true',
help='Generate reference images instead of running tests')
parser.add_argument('--skip-build',
action='store_true',
help='Skip building project before running')
additional_group = parser.add_argument_group(
'extended arguments ',
'Additional options used for testing pipelines on TeamCity.')
additional_group.add_argument(
'--pull-refs',
action='store_true',
help='Pull reference images from remote before running tests')
additional_group.add_argument(
'--push-refs',
action='store_true',
help='Push reference images to remote after generating them')
additional_group.add_argument('--build-id',
action='store',
help='TeamCity build ID',
default='unknown')
args = parser.parse_args()
# Load environment.
try:
env = Environment(args.environment, args.config)
except Exception as e:
print(e)
sys.exit(1)
# Build solution before running tests.
if not (args.skip_build or args.list):
if not build_falcor(env):
print('Build failed. Not running tests.')
sys.exit(1)
# Collect tests to run.
tests = collect_tests(env.image_tests_dir, args.filter)
if args.list:
# List available tests.
list_tests(tests)
elif args.gen_refs:
# Generate references.
ref_dir = env.resolve_image_dir(env.image_tests_ref_dir, env.branch,
args.build_id)
if not generate_refs(env, tests, ref_dir):
sys.exit(1)
# Push references to remote.
if args.push_refs:
if not env.image_tests_remote_ref_dir:
print(
"Remote reference directory is not configured for this environment."
)
sys.exit(1)
remote_ref_dir = env.resolve_image_dir(
env.image_tests_remote_ref_dir, env.branch, args.build_id)
if not push_refs(ref_dir, remote_ref_dir):
sys.exit(1)
else:
# Determine result and reference directories.
result_dir = env.resolve_image_dir(env.image_tests_result_dir,
env.branch, args.build_id)
ref_dir = env.resolve_image_dir(env.image_tests_ref_dir,
args.ref_branch, args.build_id)
# Pull references from remote.
if args.pull_refs:
if not env.image_tests_remote_ref_dir:
print(
"Remote reference directory is not configured for this environment."
)
sys.exit(1)
remote_ref_dir = env.resolve_image_dir(
env.image_tests_remote_ref_dir, args.ref_branch, args.build_id)
if not pull_refs(remote_ref_dir, ref_dir):
sys.exit(1)
# Give some instructions on how to acquire reference images if not available.
if not ref_dir.exists():
print(ref_dir)
print(
colored(
f'\n!!! Reference images for "{args.ref_branch}" branch are not available !!!',
'red'))
print('')
print(f'You have the following options:')
print('')
print(
f' 1. Checkout "{args.ref_branch}" branch and generate reference images using:'
)
print('')
print(f' run_image_tests --gen-refs')
print('')
print(f' 2. Use references from a different branch using:')
print('')
print(f' run_image_tests --ref-branch BRANCH')
print('')
sys.exit(1)
# Run tests.
if not run_tests(env, tests, args.compare_only, ref_dir, result_dir):
sys.exit(1)
sys.exit(0)
