def main(tests, root_uri, verbose):
"""Runs RV end-to-end and checks that evaluation metrics are correct."""
if len(tests) == 0:
tests = all_tests
if verbose:
rv_config.set_verbosity(verbosity=Verbosity.DEBUG)
with rv_config.get_tmp_dir() as tmp_dir:
if root_uri:
tmp_dir = root_uri
errors = []
for test in tests:
if test not in all_tests:
print('{} is not a valid test.'.format(test))
return
errors.extend(run_test(test, tmp_dir))
for error in errors:
print(error)
for test in tests:
nb_test_errors = len(
list(filter(lambda error: error.test == test, errors)))
if nb_test_errors == 0:
print('{} test passed!'.format(test))
if errors:
exit(1)
def main(tests, root_uri, verbose):
"""Runs RV end-to-end and checks that evaluation metrics are correct."""
if verbose:
rv_config.set_verbosity(verbosity=Verbosity.DEBUG)
if len(tests) == 0:
# no tests specified, so run all
tests = list(ALL_TESTS.keys())
else:
# run all tests that start with the given string e.g "chip" will match
# both "chip_classification.basic" and "chip_classification.nochip"
_tests = []
for t in tests:
t = t.strip().lower()
matching_tests = [k for k in ALL_TESTS.keys() if k.startswith(t)]
_tests.extend(matching_tests)
if len(matching_tests) == 0:
console_error(
f'{t} does not match any valid tests. Valid tests are: ')
console_error(pformat(list(ALL_TESTS.keys())))
continue
tests = _tests
console_info('The following tests will be run:')
console_info(pformat(tests, compact=False))
with rv_config.get_tmp_dir() as tmp_dir:
if root_uri:
tmp_dir = root_uri
num_failed = 0
errors = {}
for test_id in tests:
test_cfg = ALL_TESTS[test_id]
errors[test_id] = run_test(test_id, test_cfg, tmp_dir)
if len(errors[test_id]) > 0:
num_failed += 1
for error in errors[test_id]:
console_error(str(error))
for test_id in tests:
if test_id not in errors:
continue
if len(errors[test_id]) == 0:
console_success(f'{test_id}: test passed!', bold=True)
else:
console_error(f'{test_id}: test failed!', bold=True)
if num_failed > 0:
console_error(
f'Tests passed: {len(tests) - num_failed} of {len(tests)}')
console_error('Error counts:')
console_error(pformat({k: len(es) for k, es in errors.items()}))
exit(1)
def main(ctx: click.Context, profile: Optional[str], verbose: int,
tmpdir: str):
"""The main click command.
Sets the profile, verbosity, and tmp_dir in RVConfig.
"""
# Make sure current directory is on PYTHON_PATH
# so that we can run against modules in current dir.
sys.path.append(os.curdir)
rv_config.set_verbosity(verbosity=verbose + 1)
rv_config.set_tmp_dir_root(tmp_dir_root=tmpdir)
rv_config.set_everett_config(profile=profile)