def setUp(self):
self.loader = RegressionCheckLoader([
'unittests/resources/checks_unlisted/dependencies/normal.py'
])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def test_dependencies(self):
self.loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/deps_complex.py'])
# Setup the runner
self.checks = self.loader.load_all()
self.runall(self.checks, sort=True)
self.assertRunall()
stats = self.runner.stats
assert stats.num_cases(0) == 10
assert len(stats.failures()) == 4
for tf in stats.failures():
check = tf.testcase.check
_, exc_value, _ = tf.exc_info
if check.name == 'T7' or check.name == 'T9':
assert isinstance(exc_value, TaskDependencyError)
# Check that cleanup is executed properly for successful tests as well
for t in stats.tasks():
check = t.testcase.check
if t.failed:
continue
if t.ref_count == 0:
assert os.path.exists(os.path.join(check.outputdir, 'out.txt'))
def setUp(self):
get_modules_system().searchpath_add(fixtures.TEST_MODULES)
# Load a system configuration
self.system, self.partition, self.progenv = fixtures.get_test_config()
self.resourcesdir = tempfile.mkdtemp(dir='unittests')
self.loader = RegressionCheckLoader(['unittests/resources'])
self.resources = ResourcesManager(prefix=self.resourcesdir)
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources', 'unittests/foobar'], ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['badchecks'],
prefix=os.path.abspath('unittests/resources'))
self.system = System('foo')
self.resources = ResourcesManager()
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/emptycheck.py',
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/emptycheck.py'),
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources',
recurse=True,
system=self.system,
resources=self.resources)
self.assertEqual(11, len(checks))
def test_load_all(self):
checks = self.loader_with_path.load_all(system=self.system,
resources=self.resources)
self.assertEqual(10, len(checks))
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all(system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
def test_conflicted_checks(self):
self.loader_with_path._ignore_conflicts = False
self.assertRaises(NameConflictError,
self.loader_with_path.load_all,
system=self.system,
resources=self.resources)
def test_load_error(self):
self.assertRaises(OSError, self.loader.load_from_file,
'unittests/resources/foo.py')
def test_sigterm(self):
self.loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/selfkill.py']
)
checks = self.loader.load_all()
with pytest.raises(ReframeForceExitError,
match='received TERM signal'):
self.runall(checks)
self.assert_all_dead()
assert self.runner.stats.num_cases() == 1
assert len(self.runner.stats.failures()) == 1
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources/checks', 'unittests/foobar'],
ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['bad'],
prefix=os.path.abspath('unittests/resources/checks'))
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/checks/emptycheck.py')
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'EmptyTest')
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/checks/emptycheck.py'))
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'EmptyTest')
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources/checks',
recurse=True)
self.assertEqual(12, len(checks))
def test_load_all(self):
checks = self.loader_with_path.load_all()
self.assertEqual(11, len(checks))
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all()
self.assertEqual(1, len(checks))
def test_load_new_syntax(self):
checks = self.loader.load_from_file(
'unittests/resources/checks_unlisted/good.py')
self.assertEqual(13, len(checks))
def test_conflicted_checks(self):
self.loader_with_path._ignore_conflicts = False
self.assertRaises(NameConflictError, self.loader_with_path.load_all)
def test_load_error(self):
self.assertRaises(OSError, self.loader.load_from_file,
'unittests/resources/checks/foo.py')
def test_load_bad_required_version(self):
with self.assertRaises(ValueError):
self.loader.load_from_file('unittests/resources/checks_unlisted/'
'no_required_version.py')
def test_load_bad_init(self):
tests = self.loader.load_from_file(
'unittests/resources/checks_unlisted/bad_init_check.py')
self.assertEqual(0, len(tests))
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
warnings.simplefilter('ignore', ReframeDeprecationWarning)
self.loader = RegressionCheckLoader(['.'])
self.loader_with_path = RegressionCheckLoader(
['unittests/resources', 'unittests/foobar'])
self.loader_with_prefix = RegressionCheckLoader(
load_path=['badchecks'],
prefix=os.path.abspath('unittests/resources'))
self.system = System('foo')
self.resources = ResourcesManager()
def tearDown(self):
warnings.simplefilter('default', ReframeDeprecationWarning)
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/emptycheck.py',
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/emptycheck.py'),
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources',
recurse=True,
system=self.system,
resources=self.resources)
self.assertEqual(12, len(checks))
def test_load_all(self):
checks = self.loader_with_path.load_all(system=self.system,
resources=self.resources)
self.assertEqual(11, len(checks))
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all(system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
def test_load_error(self):
self.assertRaises(ReframeError, self.loader.load_from_file,
'unittests/resources/foo.py')
def setUp(self):
self.loader = RegressionCheckLoader(['unittests/resources/checks'],
ignore_conflicts=True)
# Setup the runner
self.runner = executors.Runner(policies.SerialExecutionPolicy())
self.checks = self.loader.load_all()
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
# Reset current_run
rt.runtime()._current_run = 0
def test_ci_gitlab_pipeline():
loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/deps_complex.py'])
cases = dependencies.toposort(
dependencies.build_deps(executors.generate_testcases(
loader.load_all()))[0])
with io.StringIO() as fp:
ci.emit_pipeline(fp, cases)
pipeline = fp.getvalue()
# Fetch the latest Gitlab CI JSON schema
response = requests.get('https://json.schemastore.org/gitlab-ci')
assert response.ok
schema = response.json()
jsonschema.validate(yaml.safe_load(pipeline), schema)
def setUp(self):
# Load a system configuration
settings = config.load_from_file("reframe/settings.py")
self.site_config = config.SiteConfiguration()
self.site_config.load_from_dict(settings.site_configuration)
self.system = self.site_config.systems['generic']
self.resourcesdir = tempfile.mkdtemp(dir='unittests')
self.resources = ResourcesManager(prefix=self.resourcesdir)
self.loader = RegressionCheckLoader(['unittests/resources'],
ignore_conflicts=True)
# Init modules system
init_modules_system(self.system.modules_system)
# Setup the runner
self.runner = executors.Runner(policies.SerialExecutionPolicy())
self.checks = self.loader.load_all(system=self.system,
resources=self.resources)
def loader_with_path_tmpdir(tmp_path):
test_dir_a = tmp_path / 'a'
test_dir_b = tmp_path / 'b'
os.mkdir(test_dir_a)
os.mkdir(test_dir_b)
test_a = 'unittests/resources/checks/emptycheck.py'
test_b = 'unittests/resources/checks/hellocheck.py'
shutil.copyfile(test_a, test_dir_a / 'test.py')
shutil.copyfile(test_b, test_dir_b / 'test.py')
return RegressionCheckLoader(
[test_dir_a.as_posix(), test_dir_b.as_posix()])
def test_ci_gitlab_pipeline():
loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/deps_complex.py'])
cases = dependencies.toposort(
dependencies.build_deps(executors.generate_testcases(
loader.load_all()))[0])
with io.StringIO() as fp:
ci.emit_pipeline(fp, cases)
pipeline = fp.getvalue()
# Fetch the latest Gitlab CI JSON schema
try:
response = requests.get(
'https://gitlab.com/gitlab-org/gitlab/-/raw/master/app/assets/javascripts/editor/schema/ci.json' # noqa: E501
)
except requests.exceptions.ConnectionError as e:
pytest.skip(f'could not reach URL: {e}')
else:
assert response.ok
schema = response.json()
jsonschema.validate(yaml.safe_load(pipeline), schema)
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources/checks', 'unittests/foobar'],
ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['bad'],
prefix=os.path.abspath('unittests/resources/checks'))
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources', 'unittests/foobar'], ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['badchecks'],
prefix=os.path.abspath('unittests/resources'))
self.system = System('foo')
self.resources = ResourcesManager()
def setUp(self):
warnings.simplefilter('ignore', ReframeDeprecationWarning)
self.loader = RegressionCheckLoader(['.'])
self.loader_with_path = RegressionCheckLoader(
['unittests/resources', 'unittests/foobar'])
self.loader_with_prefix = RegressionCheckLoader(
load_path=['badchecks'],
prefix=os.path.abspath('unittests/resources'))
self.system = System('foo')
self.resources = ResourcesManager()
def setUp(self):
self.partition, self.prgenv = _setup_local_execution()
self.loader = RegressionCheckLoader(['unittests/resources/checks'])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
class TestRegressionTest(unittest.TestCase):
def setUp(self):
self.partition, self.prgenv = _setup_local_execution()
self.loader = RegressionCheckLoader(['unittests/resources/checks'])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
os_ext.rmtree('.rfm_testing', ignore_errors=True)
def replace_prefix(self, filename, new_prefix):
basename = os.path.basename(filename)
return os.path.join(new_prefix, basename)
def keep_files_list(self, test, compile_only=False):
ret = [
self.replace_prefix(sn.evaluate(test.stdout), test.outputdir),
self.replace_prefix(sn.evaluate(test.stderr), test.outputdir)
]
if not compile_only:
ret.append(
self.replace_prefix(test.job.script_filename, test.outputdir))
ret.extend(
[self.replace_prefix(f, test.outputdir) for f in test.keep_files])
return ret
def test_environ_setup(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
test.modules = ['testmod_foo']
test.variables = {'_FOO_': '1', '_BAR_': '2'}
test.local = True
test.setup(self.partition, self.prgenv)
for k in test.variables.keys():
assert k not in os.environ
def _run_test(self, test, compile_only=False):
_run(test, self.partition, self.prgenv)
assert not os.path.exists(test.stagedir)
for f in self.keep_files_list(test, compile_only):
assert os.path.exists(f)
@fixtures.switch_to_user_runtime
def test_hellocheck(self):
self.partition, self.prgenv = _setup_remote_execution()
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
self._run_test(test)
@fixtures.switch_to_user_runtime
def test_hellocheck_make(self):
self.partition, self.prgenv = _setup_remote_execution()
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck_make.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
self._run_test(test)
def test_hellocheck_local(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
# Test also the prebuild/postbuild functionality
test.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir']
test.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir']
test.keep_files = [
'prebuild', 'postbuild', 'prebuild_dir', 'postbuild_dir'
]
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_prepost_run(self):
@sn.sanity_function
def stagedir(test):
return test.stagedir
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
# Test also the prebuild/postbuild functionality
test.pre_run = ['echo prerun: `pwd`']
test.post_run = ['echo postrun: `pwd`']
pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1)
post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1)
test.sanity_patterns = sn.all([
sn.assert_eq(stagedir(test), pre_run_path),
sn.assert_eq(stagedir(test), post_run_path),
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_prepost_run_in_setup(self):
def custom_setup(obj, partition, environ, **job_opts):
super(obj.__class__, obj).setup(partition, environ, **job_opts)
obj.pre_run = ['echo Prerunning cmd from setup phase']
obj.post_run = ['echo Postruning cmd from setup phase']
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Monkey patch the setup method of the test
test.setup = custom_setup.__get__(test)
# Use test environment for the regression check
test.valid_prog_environs = ['*']
test.sanity_patterns = sn.all([
sn.assert_found(r'^Prerunning cmd from setup phase', test.stdout),
sn.assert_found(r'Hello, World\!', test.stdout),
sn.assert_found(r'^Postruning cmd from setup phase', test.stdout)
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_run_only_sanity(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.executable = './hello.sh'
self.executable_opts = ['Hello, World!']
self.local = True
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_run_only_no_srcdir(self):
@fixtures.custom_prefix('foo/bar/')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.executable = 'echo'
self.executable_opts = ['hello']
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'hello', self.stdout)
test = MyTest()
assert test.sourcesdir is None
self._run_test(MyTest())
def test_compile_only_failure(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.sourcepath = 'compiler_failure.c'
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
test = MyTest()
test.setup(self.partition, self.prgenv)
test.compile()
with pytest.raises(BuildError):
test.compile_wait()
def test_compile_only_warning(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.build_system = 'SingleSource'
self.build_system.srcfile = 'compiler_warning.c'
self.build_system.cflags = ['-Wall']
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'warning', self.stderr)
self._run_test(MyTest(), compile_only=True)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
def test_supports_system(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
test.valid_systems = ['*']
assert test.supports_system('gpu')
assert test.supports_system('login')
assert test.supports_system('testsys:gpu')
assert test.supports_system('testsys:login')
test.valid_systems = ['testsys']
assert test.supports_system('gpu')
assert test.supports_system('login')
assert test.supports_system('testsys:gpu')
assert test.supports_system('testsys:login')
test.valid_systems = ['testsys:gpu']
assert test.supports_system('gpu')
assert not test.supports_system('login')
assert test.supports_system('testsys:gpu')
assert not test.supports_system('testsys:login')
test.valid_systems = ['testsys:login']
assert not test.supports_system('gpu')
assert test.supports_system('login')
assert not test.supports_system('testsys:gpu')
assert test.supports_system('testsys:login')
test.valid_systems = ['foo']
assert not test.supports_system('gpu')
assert not test.supports_system('login')
assert not test.supports_system('testsys:gpu')
assert not test.supports_system('testsys:login')
def test_supports_environ(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
test.valid_prog_environs = ['*']
assert test.supports_environ('foo1')
assert test.supports_environ('foo-env')
assert test.supports_environ('*')
def test_sourcesdir_none(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RegressionTest):
def __init__(self):
self.sourcesdir = None
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
with pytest.raises(ReframeError):
self._run_test(MyTest())
def test_sourcesdir_build_system(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RegressionTest):
def __init__(self):
self.build_system = 'Make'
self.sourcepath = 'code'
self.executable = './code/hello'
self.local = True
self.valid_systems = ['*']
self.valid_prog_environs = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_sourcesdir_none_generated_sources(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RegressionTest):
def __init__(self):
self.sourcesdir = None
self.prebuild_cmd = [
"printf '#include <stdio.h>\\n int main(){ "
"printf(\"Hello, World!\\\\n\"); return 0; }' > hello.c"
]
self.executable = './hello'
self.sourcepath = 'hello.c'
self.local = True
self.valid_systems = ['*']
self.valid_prog_environs = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_sourcesdir_none_compile_only(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.sourcesdir = None
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
with pytest.raises(BuildError):
self._run_test(MyTest())
def test_sourcesdir_none_run_only(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.sourcesdir = None
self.executable = 'echo'
self.executable_opts = ["Hello, World!"]
self.local = True
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_sourcepath_abs(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
test = MyTest()
test.setup(self.partition, self.prgenv)
test.sourcepath = '/usr/src'
with pytest.raises(PipelineError):
test.compile()
def test_sourcepath_upref(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
test = MyTest()
test.setup(self.partition, self.prgenv)
test.sourcepath = '../hellosrc'
with pytest.raises(PipelineError):
test.compile()
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
def test_extra_resources(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(HelloTest):
def __init__(self):
super().__init__()
self.name = type(self).__name__
self.executable = os.path.join('.', self.name)
self.local = True
@rfm.run_after('setup')
def set_resources(self):
test.extra_resources = {
'gpu': {
'num_gpus_per_node': 2
},
'datawarp': {
'capacity': '100GB',
'stagein_src': test.stagedir
}
}
test.job.options += ['--foo']
test = MyTest()
partition = rt.runtime().system.partition('gpu')
environ = partition.environment('builtin-gcc')
_run(test, partition, environ)
expected_job_options = [
'--gres=gpu:2', '#DW jobdw capacity=100GB',
'#DW stage_in source=%s' % test.stagedir, '--foo'
]
self.assertCountEqual(expected_job_options, test.job.options)
class TestSerialExecutionPolicy(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['unittests/resources/checks'],
ignore_conflicts=True)
# Setup the runner
self.runner = executors.Runner(policies.SerialExecutionPolicy())
self.checks = self.loader.load_all()
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
# Reset current_run
rt.runtime()._current_run = 0
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
def _num_failures_stage(self, stage):
stats = self.runner.stats
return len(
[t for t in stats.tasks_failed() if t.failed_stage == stage])
def assert_all_dead(self):
stats = self.runner.stats
for t in self.runner.stats.get_tasks():
try:
finished = t.check.poll()
except JobNotStartedError:
finished = True
self.assertTrue(finished)
def test_runall(self):
self.runner.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(4, stats.num_failures())
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_system_check(self):
self.runner.policy.skip_system_check = True
self.runner.runall(self.checks)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(4, stats.num_failures())
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_prgenv_check(self):
self.runner.policy.skip_environ_check = True
self.runner.runall(self.checks)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(4, stats.num_failures())
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_sanity_check(self):
self.runner.policy.skip_sanity_check = True
self.runner.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(3, stats.num_failures())
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(0, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_performance_check(self):
self.runner.policy.skip_performance_check = True
self.runner.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(3, stats.num_failures())
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(0, self._num_failures_stage('performance'))
def test_strict_performance_check(self):
self.runner.policy.strict_check = True
self.runner.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(5, stats.num_failures())
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(2, self._num_failures_stage('performance'))
def test_force_local_execution(self):
self.runner.policy.force_local = True
self.runner.runall([HelloTest()])
stats = self.runner.stats
for t in stats.get_tasks():
self.assertTrue(t.check.local)
def test_kbd_interrupt_within_test(self):
check = KeyboardInterruptCheck()
self.assertRaises(KeyboardInterrupt, self.runner.runall, [check])
stats = self.runner.stats
self.assertEqual(1, stats.num_failures())
self.assert_all_dead()
def test_system_exit_within_test(self):
check = SystemExitCheck()
# This should not raise and should not exit
self.runner.runall([check])
stats = self.runner.stats
self.assertEqual(1, stats.num_failures())
def test_retries_bad_check(self):
max_retries = 2
checks = [BadSetupCheck(), BadSetupCheckEarly()]
self.runner._max_retries = max_retries
self.runner.runall(checks)
# Ensure that the test was retried #max_retries times and failed.
self.assertEqual(2, self.runner.stats.num_cases())
self.assertEqual(max_retries, rt.runtime().current_run)
self.assertEqual(2, self.runner.stats.num_failures())
# Ensure that the report does not raise any exception.
self.runner.stats.retry_report()
def test_retries_good_check(self):
max_retries = 2
checks = [HelloTest()]
self.runner._max_retries = max_retries
self.runner.runall(checks)
# Ensure that the test passed without retries.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(0, rt.runtime().current_run)
self.assertEqual(0, self.runner.stats.num_failures())
def test_pass_in_retries(self):
max_retries = 3
run_to_pass = 2
# Create a file containing the current_run; Run 0 will set it to 0,
# run 1 to 1 and so on.
with tempfile.NamedTemporaryFile(mode='wt', delete=False) as fp:
fp.write('0\n')
checks = [RetriesCheck(run_to_pass, fp.name)]
self.runner._max_retries = max_retries
self.runner.runall(checks)
# Ensure that the test passed after retries in run #run_to_pass.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(1, self.runner.stats.num_failures(run=0))
self.assertEqual(run_to_pass, rt.runtime().current_run)
self.assertEqual(0, self.runner.stats.num_failures())
os.remove(fp.name)
class TestDependencies(unittest.TestCase):
class Node:
'''A node in the test case graph.
It's simply a wrapper to a (test_name, partition, environment) tuple
that can interact seemlessly with a real test case.
It's meant for convenience in unit testing.
'''
def __init__(self, cname, pname, ename):
self.cname, self.pname, self.ename = cname, pname, ename
def __eq__(self, other):
if isinstance(other, type(self)):
return (self.cname == other.cname and self.pname == other.pname
and self.ename == other.ename)
if isinstance(other, executors.TestCase):
return (self.cname == other.check.name
and self.pname == other.partition.fullname
and self.ename == other.environ.name)
return NotImplemented
def __hash__(self):
return hash(self.cname) ^ hash(self.pname) ^ hash(self.ename)
def __repr__(self):
return 'Node(%r, %r, %r)' % (self.cname, self.pname, self.ename)
def has_edge(graph, src, dst):
return dst in graph[src]
def num_deps(graph, cname):
return sum(
len(deps) for c, deps in graph.items() if c.check.name == cname)
def in_degree(graph, node):
for v in graph.keys():
if v == node:
return v.num_dependents
def find_check(name, checks):
for c in checks:
if c.name == name:
return c
return None
def find_case(cname, ename, cases):
for c in cases:
if c.check.name == cname and c.environ.name == ename:
return c
def setUp(self):
self.loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/deps_simple.py'])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_eq_hash(self):
find_case = TestDependencies.find_case
cases = executors.generate_testcases(self.loader.load_all())
case0 = find_case('Test0', 'e0', cases)
case1 = find_case('Test0', 'e1', cases)
case0_copy = case0.clone()
assert case0 == case0_copy
assert hash(case0) == hash(case0_copy)
assert case1 != case0
assert hash(case1) != hash(case0)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_build_deps(self):
Node = TestDependencies.Node
has_edge = TestDependencies.has_edge
num_deps = TestDependencies.num_deps
in_degree = TestDependencies.in_degree
find_check = TestDependencies.find_check
find_case = TestDependencies.find_case
checks = self.loader.load_all()
cases = executors.generate_testcases(checks)
# Test calling getdep() before having built the graph
t = find_check('Test1_exact', checks)
with pytest.raises(DependencyError):
t.getdep('Test0', 'e0')
# Build dependencies and continue testing
deps = dependency.build_deps(cases)
dependency.validate_deps(deps)
# Check DEPEND_FULLY dependencies
assert num_deps(deps, 'Test1_fully') == 8
for p in ['sys0:p0', 'sys0:p1']:
for e0 in ['e0', 'e1']:
for e1 in ['e0', 'e1']:
assert has_edge(deps, Node('Test1_fully', p, e0),
Node('Test0', p, e1))
# Check DEPEND_BY_ENV
assert num_deps(deps, 'Test1_by_env') == 4
assert num_deps(deps, 'Test1_default') == 4
for p in ['sys0:p0', 'sys0:p1']:
for e in ['e0', 'e1']:
assert has_edge(deps, Node('Test1_by_env', p, e),
Node('Test0', p, e))
assert has_edge(deps, Node('Test1_default', p, e),
Node('Test0', p, e))
# Check DEPEND_EXACT
assert num_deps(deps, 'Test1_exact') == 6
for p in ['sys0:p0', 'sys0:p1']:
assert has_edge(deps, Node('Test1_exact', p, 'e0'),
Node('Test0', p, 'e0'))
assert has_edge(deps, Node('Test1_exact', p, 'e0'),
Node('Test0', p, 'e1'))
assert has_edge(deps, Node('Test1_exact', p, 'e1'),
Node('Test0', p, 'e1'))
# Check in-degree of Test0
# 2 from Test1_fully,
# 1 from Test1_by_env,
# 1 from Test1_exact,
# 1 from Test1_default
assert in_degree(deps, Node('Test0', 'sys0:p0', 'e0')) == 5
assert in_degree(deps, Node('Test0', 'sys0:p1', 'e0')) == 5
# 2 from Test1_fully,
# 1 from Test1_by_env,
# 2 from Test1_exact,
# 1 from Test1_default
assert in_degree(deps, Node('Test0', 'sys0:p0', 'e1')) == 6
assert in_degree(deps, Node('Test0', 'sys0:p1', 'e1')) == 6
# Pick a check to test getdep()
check_e0 = find_case('Test1_exact', 'e0', cases).check
check_e1 = find_case('Test1_exact', 'e1', cases).check
with pytest.raises(DependencyError):
check_e0.getdep('Test0')
# Set the current environment
check_e0._current_environ = Environment('e0')
check_e1._current_environ = Environment('e1')
assert check_e0.getdep('Test0', 'e0').name == 'Test0'
assert check_e0.getdep('Test0', 'e1').name == 'Test0'
assert check_e1.getdep('Test0', 'e1').name == 'Test0'
with pytest.raises(DependencyError):
check_e0.getdep('TestX', 'e0')
with pytest.raises(DependencyError):
check_e0.getdep('Test0', 'eX')
with pytest.raises(DependencyError):
check_e1.getdep('Test0', 'e0')
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_build_deps_unknown_test(self):
find_check = TestDependencies.find_check
checks = self.loader.load_all()
# Add some inexistent dependencies
test0 = find_check('Test0', checks)
for depkind in ('default', 'fully', 'by_env', 'exact'):
test1 = find_check('Test1_' + depkind, checks)
if depkind == 'default':
test1.depends_on('TestX')
elif depkind == 'exact':
test1.depends_on('TestX', rfm.DEPEND_EXACT, {'e0': ['e0']})
elif depkind == 'fully':
test1.depends_on('TestX', rfm.DEPEND_FULLY)
elif depkind == 'by_env':
test1.depends_on('TestX', rfm.DEPEND_BY_ENV)
with pytest.raises(DependencyError):
dependency.build_deps(executors.generate_testcases(checks))
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_build_deps_unknown_target_env(self):
find_check = TestDependencies.find_check
checks = self.loader.load_all()
# Add some inexistent dependencies
test0 = find_check('Test0', checks)
test1 = find_check('Test1_default', checks)
test1.depends_on('Test0', rfm.DEPEND_EXACT, {'e0': ['eX']})
with pytest.raises(DependencyError):
dependency.build_deps(executors.generate_testcases(checks))
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_build_deps_unknown_source_env(self):
find_check = TestDependencies.find_check
num_deps = TestDependencies.num_deps
checks = self.loader.load_all()
# Add some inexistent dependencies
test0 = find_check('Test0', checks)
test1 = find_check('Test1_default', checks)
test1.depends_on('Test0', rfm.DEPEND_EXACT, {'eX': ['e0']})
# Unknown source is ignored, because it might simply be that the test
# is not executed for eX
deps = dependency.build_deps(executors.generate_testcases(checks))
assert num_deps(deps, 'Test1_default') == 4
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_build_deps_empty(self):
assert {} == dependency.build_deps([])
def create_test(self, name):
test = rfm.RegressionTest()
test.name = name
test.valid_systems = ['*']
test.valid_prog_environs = ['*']
test.executable = 'echo'
test.executable_opts = [name]
return test
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_valid_deps(self):
#
# t0 +-->t5<--+
# ^ | |
# | | |
# +-->t1<--+ t6 t7
# | | ^
# t2<------t3 |
# ^ ^ |
# | | t8
# +---t4---+
#
t0 = self.create_test('t0')
t1 = self.create_test('t1')
t2 = self.create_test('t2')
t3 = self.create_test('t3')
t4 = self.create_test('t4')
t5 = self.create_test('t5')
t6 = self.create_test('t6')
t7 = self.create_test('t7')
t8 = self.create_test('t8')
t1.depends_on('t0')
t2.depends_on('t1')
t3.depends_on('t1')
t3.depends_on('t2')
t4.depends_on('t2')
t4.depends_on('t3')
t6.depends_on('t5')
t7.depends_on('t5')
t8.depends_on('t7')
dependency.validate_deps(
dependency.build_deps(
executors.generate_testcases(
[t0, t1, t2, t3, t4, t5, t6, t7, t8])))
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_cyclic_deps(self):
#
# t0 +-->t5<--+
# ^ | |
# | | |
# +-->t1<--+ t6 t7
# | | | ^
# t2 | t3 |
# ^ | ^ |
# | v | t8
# +---t4---+
#
t0 = self.create_test('t0')
t1 = self.create_test('t1')
t2 = self.create_test('t2')
t3 = self.create_test('t3')
t4 = self.create_test('t4')
t5 = self.create_test('t5')
t6 = self.create_test('t6')
t7 = self.create_test('t7')
t8 = self.create_test('t8')
t1.depends_on('t0')
t1.depends_on('t4')
t2.depends_on('t1')
t3.depends_on('t1')
t4.depends_on('t2')
t4.depends_on('t3')
t6.depends_on('t5')
t7.depends_on('t5')
t8.depends_on('t7')
deps = dependency.build_deps(
executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8]))
with pytest.raises(DependencyError) as exc_info:
dependency.validate_deps(deps)
assert ('t4->t2->t1->t4' in str(exc_info.value)
or 't2->t1->t4->t2' in str(exc_info.value)
or 't1->t4->t2->t1' in str(exc_info.value)
or 't1->t4->t3->t1' in str(exc_info.value)
or 't4->t3->t1->t4' in str(exc_info.value)
or 't3->t1->t4->t3' in str(exc_info.value))
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_cyclic_deps_by_env(self):
t0 = self.create_test('t0')
t1 = self.create_test('t1')
t1.depends_on('t0', rfm.DEPEND_EXACT, {'e0': ['e0']})
t0.depends_on('t1', rfm.DEPEND_EXACT, {'e1': ['e1']})
deps = dependency.build_deps(executors.generate_testcases([t0, t1]))
with pytest.raises(DependencyError) as exc_info:
dependency.validate_deps(deps)
assert ('t1->t0->t1' in str(exc_info.value)
or 't0->t1->t0' in str(exc_info.value))
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_validate_deps_empty(self):
dependency.validate_deps({})
def assert_topological_order(self, cases, graph):
cases_order = []
visited_tests = set()
tests = util.OrderedSet()
for c in cases:
check, part, env = c
cases_order.append((check.name, part.fullname, env.name))
tests.add(check.name)
visited_tests.add(check.name)
# Assert that all dependencies of c have been visited before
for d in graph[c]:
if d not in cases:
# dependency points outside the subgraph
continue
assert d.check.name in visited_tests
# Check the order of systems and prog. environments
# We are checking against all possible orderings
valid_orderings = []
for partitions in itertools.permutations(['sys0:p0', 'sys0:p1']):
for environs in itertools.permutations(['e0', 'e1']):
ordering = []
for t in tests:
for p in partitions:
for e in environs:
ordering.append((t, p, e))
valid_orderings.append(ordering)
assert cases_order in valid_orderings
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_toposort(self):
#
# t0 +-->t5<--+
# ^ | |
# | | |
# +-->t1<--+ t6 t7
# | | ^
# t2<------t3 |
# ^ ^ |
# | | t8
# +---t4---+
#
t0 = self.create_test('t0')
t1 = self.create_test('t1')
t2 = self.create_test('t2')
t3 = self.create_test('t3')
t4 = self.create_test('t4')
t5 = self.create_test('t5')
t6 = self.create_test('t6')
t7 = self.create_test('t7')
t8 = self.create_test('t8')
t1.depends_on('t0')
t2.depends_on('t1')
t3.depends_on('t1')
t3.depends_on('t2')
t4.depends_on('t2')
t4.depends_on('t3')
t6.depends_on('t5')
t7.depends_on('t5')
t8.depends_on('t7')
deps = dependency.build_deps(
executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8]))
cases = dependency.toposort(deps)
self.assert_topological_order(cases, deps)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'sys0')
def test_toposort_subgraph(self):
#
# t0
# ^
# |
# +-->t1<--+
# | |
# t2<------t3
# ^ ^
# | |
# +---t4---+
#
t0 = self.create_test('t0')
t1 = self.create_test('t1')
t2 = self.create_test('t2')
t3 = self.create_test('t3')
t4 = self.create_test('t4')
t1.depends_on('t0')
t2.depends_on('t1')
t3.depends_on('t1')
t3.depends_on('t2')
t4.depends_on('t2')
t4.depends_on('t3')
full_deps = dependency.build_deps(
executors.generate_testcases([t0, t1, t2, t3, t4]))
partial_deps = dependency.build_deps(
executors.generate_testcases([t3, t4]), full_deps)
cases = dependency.toposort(partial_deps, is_subgraph=True)
self.assert_topological_order(cases, partial_deps)
class TestSerialExecutionPolicy(unittest.TestCase):
def setUp(self):
# Load a system configuration
settings = config.load_from_file("reframe/settings.py")
self.site_config = config.SiteConfiguration()
self.site_config.load_from_dict(settings.site_configuration)
self.system = self.site_config.systems['generic']
self.resourcesdir = tempfile.mkdtemp(dir='unittests')
self.resources = ResourcesManager(prefix=self.resourcesdir)
self.loader = RegressionCheckLoader(['unittests/resources'],
ignore_conflicts=True)
# Init modules system
init_modules_system(self.system.modules_system)
# Setup the runner
self.runner = executors.Runner(policies.SerialExecutionPolicy())
self.checks = self.loader.load_all(system=self.system,
resources=self.resources)
def tearDown(self):
shutil.rmtree(self.resourcesdir, ignore_errors=True)
def _num_failures_stage(self, stage):
stats = self.runner.stats
return len(
[t for t in stats.tasks_failed() if t.failed_stage == stage])
def assert_all_dead(self):
stats = self.runner.stats
for t in self.runner.stats.get_tasks():
try:
finished = t.check.poll()
except JobNotStartedError:
finished = True
self.assertTrue(finished)
def test_runall(self):
self.runner.runall(self.checks, self.system)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(5, stats.num_failures())
self.assertEqual(3, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_system_check(self):
self.runner.policy.skip_system_check = True
self.runner.runall(self.checks, self.system)
stats = self.runner.stats
self.assertEqual(9, stats.num_cases())
self.assertEqual(5, stats.num_failures())
self.assertEqual(3, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_prgenv_check(self):
self.runner.policy.skip_environ_check = True
self.runner.runall(self.checks, self.system)
stats = self.runner.stats
self.assertEqual(9, stats.num_cases())
self.assertEqual(5, stats.num_failures())
self.assertEqual(3, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_sanity_check(self):
self.runner.policy.skip_sanity_check = True
self.runner.runall(self.checks, self.system)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(4, stats.num_failures())
self.assertEqual(3, self._num_failures_stage('setup'))
self.assertEqual(0, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_performance_check(self):
self.runner.policy.skip_performance_check = True
self.runner.runall(self.checks, self.system)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(4, stats.num_failures())
self.assertEqual(3, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(0, self._num_failures_stage('performance'))
def test_strict_performance_check(self):
self.runner.policy.strict_check = True
self.runner.runall(self.checks, self.system)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(6, stats.num_failures())
self.assertEqual(3, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(2, self._num_failures_stage('performance'))
def test_force_local_execution(self):
self.runner.policy.force_local = True
self.runner.runall(
[HelloTest(system=self.system, resources=self.resources)],
self.system)
stats = self.runner.stats
for t in stats.get_tasks():
self.assertTrue(t.check.local)
def test_kbd_interrupt_within_test(self):
check = KeyboardInterruptCheck(system=self.system,
resources=self.resources)
self.assertRaises(KeyboardInterrupt, self.runner.runall, [check],
self.system)
stats = self.runner.stats
self.assertEqual(1, stats.num_failures())
self.assert_all_dead()
def test_system_exit_within_test(self):
check = SystemExitCheck(system=self.system, resources=self.resources)
# This should not raise and should not exit
self.runner.runall([check], self.system)
stats = self.runner.stats
self.assertEqual(1, stats.num_failures())
def test_retries_bad_check(self):
max_retries = 2
checks = [BadSetupCheck(system=self.system, resources=self.resources)]
self.runner._max_retries = max_retries
self.runner.runall(checks, self.system)
# Ensure that the test was retried #max_retries times and failed.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(max_retries, self.runner.stats.current_run)
self.assertEqual(1, self.runner.stats.num_failures())
def test_retries_good_check(self):
max_retries = 2
checks = [HelloTest(system=self.system, resources=self.resources)]
self.runner._max_retries = max_retries
self.runner.runall(checks, self.system)
# Ensure that the test passed without retries.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(0, self.runner.stats.current_run)
self.assertEqual(0, self.runner.stats.num_failures())
def test_pass_in_retries(self):
max_retries = 3
run_to_pass = 2
# Create a file containing the current_run; Run 0 will set it to 0,
# run 1 to 1 and so on.
with tempfile.NamedTemporaryFile(mode='wt', delete=False) as fp:
fp.write('0\n')
checks = [
RetriesCheck(run_to_pass,
fp.name,
system=self.system,
resources=self.resources)
]
self.runner._max_retries = max_retries
self.runner.runall(checks, self.system)
# Ensure that the test passed after retries in run #run_to_pass.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(1, self.runner.stats.num_failures(run=0))
self.assertEqual(run_to_pass, self.runner.stats.current_run)
self.assertEqual(0, self.runner.stats.num_failures())
os.remove(fp.name)
def _make_loader(check_search_path):
return RegressionCheckLoader(check_search_path, ignore_conflicts=True)
def main():
# Setup command line options
argparser = argparse.ArgumentParser()
output_options = argparser.add_argument_group('Options controlling output')
locate_options = argparser.add_argument_group(
'Options for locating checks')
select_options = argparser.add_argument_group(
'Options for selecting checks')
action_options = argparser.add_argument_group(
'Options controlling actions')
run_options = argparser.add_argument_group(
'Options controlling execution of checks')
env_options = argparser.add_argument_group(
'Options controlling environment')
misc_options = argparser.add_argument_group('Miscellaneous options')
# Output directory options
output_options.add_argument('--prefix',
action='store',
metavar='DIR',
help='Set output directory prefix to DIR',
envvar='RFM_PREFIX',
configvar='systems/prefix')
output_options.add_argument('-o',
'--output',
action='store',
metavar='DIR',
help='Set output directory to DIR',
envvar='RFM_OUTPUT_DIR',
configvar='systems/outputdir')
output_options.add_argument('-s',
'--stage',
action='store',
metavar='DIR',
help='Set stage directory to DIR',
envvar='RFM_STAGE_DIR',
configvar='systems/stagedir')
output_options.add_argument(
'--perflogdir',
action='store',
metavar='DIR',
help=('Set directory prefix for the performance logs '
'(default: ${prefix}/perflogs, '
'relevant only if the filelog backend is used)'),
envvar='RFM_PERFLOG_DIR',
configvar='logging/handlers_perflog/filelog_basedir')
output_options.add_argument(
'--keep-stage-files',
action='store_true',
help='Keep stage directory even if check is successful',
envvar='RFM_KEEP_STAGE_FILES',
configvar='general/keep_stage_files')
output_options.add_argument(
'--save-log-files',
action='store_true',
default=False,
help=('Copy the log file from the current directory to the '
'output directory when ReFrame ends'),
envvar='RFM_SAVE_LOG_FILES',
configvar='general/save_log_files')
# Check discovery options
locate_options.add_argument(
'-c',
'--checkpath',
action='append',
metavar='DIR|FILE',
help="Add DIR or FILE to the check search path",
envvar='RFM_CHECK_SEARCH_PATH :',
configvar='general/check_search_path')
locate_options.add_argument('-R',
'--recursive',
action='store_true',
help='Load checks recursively',
envvar='RFM_CHECK_SEARCH_RECURSIVE',
configvar='general/check_search_recursive')
locate_options.add_argument('--ignore-check-conflicts',
action='store_true',
help='Skip checks with conflicting names',
envvar='RFM_IGNORE_CHECK_CONFLICTS',
configvar='general/ignore_check_conflicts')
# Select options
select_options.add_argument('-t',
'--tag',
action='append',
dest='tags',
default=[],
help='Select checks matching TAG')
select_options.add_argument('-n',
'--name',
action='append',
dest='names',
default=[],
metavar='NAME',
help='Select checks with NAME')
select_options.add_argument('-x',
'--exclude',
action='append',
dest='exclude_names',
metavar='NAME',
default=[],
help='Exclude checks with NAME')
select_options.add_argument(
'-p',
'--prgenv',
action='append',
default=[r'.*'],
help='Select tests for PRGENV programming environment only')
select_options.add_argument('--gpu-only',
action='store_true',
help='Select only GPU tests')
select_options.add_argument('--cpu-only',
action='store_true',
help='Select only CPU tests')
# Action options
action_options.add_argument('-l',
'--list',
action='store_true',
help='List matched regression checks')
action_options.add_argument(
'-L',
'--list-detailed',
action='store_true',
help='List matched regression checks with a detailed description')
action_options.add_argument('-r',
'--run',
action='store_true',
help='Run regression with the selected checks')
# Run options
run_options.add_argument('-A',
'--account',
action='store',
help='Use ACCOUNT for submitting jobs')
run_options.add_argument('-P',
'--partition',
action='store',
metavar='PART',
help='Use PART for submitting jobs')
run_options.add_argument('--reservation',
action='store',
metavar='RES',
help='Use RES for submitting jobs')
run_options.add_argument('--nodelist',
action='store',
help='Run checks on the selected list of nodes')
run_options.add_argument(
'--exclude-nodes',
action='store',
metavar='NODELIST',
help='Exclude the list of nodes from running checks')
run_options.add_argument('--job-option',
action='append',
metavar='OPT',
dest='job_options',
default=[],
help='Pass OPT to job scheduler')
run_options.add_argument('--force-local',
action='store_true',
help='Force local execution of checks')
run_options.add_argument('--skip-sanity-check',
action='store_true',
help='Skip sanity checking')
run_options.add_argument('--skip-performance-check',
action='store_true',
help='Skip performance checking')
run_options.add_argument('--strict',
action='store_true',
help='Force strict performance checking')
run_options.add_argument('--skip-system-check',
action='store_true',
help='Skip system check')
run_options.add_argument('--skip-prgenv-check',
action='store_true',
help='Skip prog. environment check')
run_options.add_argument(
'--exec-policy',
metavar='POLICY',
action='store',
choices=['async', 'serial'],
default='async',
help='Specify the execution policy for running the regression tests. '
'Available policies: "async" (default), "serial"')
run_options.add_argument('--mode',
action='store',
help='Execution mode to use')
run_options.add_argument(
'--max-retries',
metavar='NUM',
action='store',
default=0,
help='Specify the maximum number of times a failed regression test '
'may be retried (default: 0)')
run_options.add_argument(
'--flex-alloc-tasks',
action='store',
dest='flex_alloc_tasks',
metavar='{all|idle|NUM}',
default=None,
help='*deprecated*, please use --flex-alloc-nodes instead')
run_options.add_argument(
'--flex-alloc-nodes',
action='store',
dest='flex_alloc_nodes',
metavar='{all|idle|NUM}',
default=None,
help="Strategy for flexible node allocation (default: 'idle').")
env_options.add_argument('-M',
'--map-module',
action='append',
metavar='MAPPING',
dest='module_mappings',
default=[],
help='Apply a single module mapping',
envvar='RFM_MODULE_MAPPINGS ,',
configvar='general/module_mappings')
env_options.add_argument(
'-m',
'--module',
action='append',
default=[],
metavar='MOD',
dest='user_modules',
help='Load module MOD before running the regression suite',
envvar='RFM_USER_MODULES',
configvar='general/user_modules')
env_options.add_argument('--module-mappings',
action='store',
metavar='FILE',
dest='module_map_file',
help='Apply module mappings defined in FILE',
envvar='RFM_MODULE_MAP_FILE',
configvar='general/module_map_file')
env_options.add_argument(
'-u',
'--unload-module',
action='append',
metavar='MOD',
dest='unload_modules',
default=[],
help='Unload module MOD before running the regression suite',
envvar='RFM_UNLOAD_MODULES',
configvar='general/unload_modules')
env_options.add_argument(
'--purge-env',
action='store_true',
dest='purge_env',
default=False,
help='Purge environment before running the regression suite',
envvar='RFM_PURGE_ENVIRONMENT',
configvar='general/purge_environment')
env_options.add_argument('--non-default-craype',
action='store_true',
help='Test a non-default Cray PE',
envvar='RFM_NON_DEFAULT_CRAYPE',
configvar='general/non_default_craype')
# Miscellaneous options
misc_options.add_argument('-C',
'--config-file',
action='store',
dest='config_file',
metavar='FILE',
help='ReFrame configuration file to use',
envvar='RFM_CONFIG_FILE')
misc_options.add_argument('--nocolor',
action='store_false',
dest='colorize',
help='Disable coloring of output',
envvar='RFM_COLORIZE',
configvar='general/colorize')
misc_options.add_argument('--failure-stats',
action='store_true',
help='Print failure statistics')
misc_options.add_argument('--performance-report',
action='store_true',
help='Print a report for performance tests run')
misc_options.add_argument('--show-config-param',
action='store',
nargs='?',
const='all',
metavar='PARAM',
help=('Print how parameter PARAM is configured '
'for the current system and exit'))
misc_options.add_argument('--system',
action='store',
help='Load configuration for SYSTEM',
envvar='RFM_SYSTEM')
misc_options.add_argument(
'--timestamp',
action='store',
nargs='?',
const='',
metavar='TIMEFMT',
help=('Append a timestamp component to the various '
'ReFrame directories (default format: "%%FT%%T")'),
envvar='RFM_TIMESTAMP_DIRS',
configvar='general/timestamp_dirs')
misc_options.add_argument('-V',
'--version',
action='version',
version=os_ext.reframe_version())
misc_options.add_argument('-v',
'--verbose',
action='count',
help='Increase verbosity level of output',
envvar='RFM_VERBOSE',
configvar='general/verbose')
# Options not associated with command-line arguments
argparser.add_argument(
dest='graylog_server',
envvar='RFM_GRAYLOG_SERVER',
configvar='logging/handlers_perflog/graylog_address',
help='Graylog server address')
if len(sys.argv) == 1:
argparser.print_help()
sys.exit(1)
# Parse command line
options = argparser.parse_args()
# First configure logging with our generic configuration so as to be able
# to print pretty messages; logging will be reconfigured by user's
# configuration later
site_config = config.load_config(
os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py'))
site_config.select_subconfig('generic')
options.update_config(site_config)
logging.configure_logging(site_config)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer = PrettyPrinter()
printer.colorize = site_config.get('general/0/colorize')
if options.verbose:
printer.inc_verbosity(options.verbose)
# Now configure ReFrame according to the user configuration file
try:
try:
site_config = config.load_config(options.config_file)
except ReframeDeprecationWarning as e:
printer.warning(e)
converted = config.convert_old_config(options.config_file)
printer.warning(f"configuration file has been converted "
f"to the new syntax here: '{converted}'")
site_config = config.load_config(converted)
site_config.validate()
site_config.select_subconfig(options.system)
for err in options.update_config(site_config):
printer.warning(str(err))
logging.configure_logging(site_config)
except (OSError, ConfigError) as e:
printer.error(f'failed to load configuration: {e}')
sys.exit(1)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer.colorize = site_config.get('general/0/colorize')
try:
runtime.init_runtime(site_config)
except ConfigError as e:
printer.error(f'failed to initialize runtime: {e}')
sys.exit(1)
rt = runtime.runtime()
try:
if site_config.get('general/0/module_map_file'):
rt.modules_system.load_mapping_from_file(
site_config.get('general/0/module_map_file'))
if site_config.get('general/0/module_mappings'):
for m in site_config.get('general/0/module_mappings'):
rt.modules_system.load_mapping(m)
except (ConfigError, OSError) as e:
printer.error('could not load module mappings: %s' % e)
sys.exit(1)
if options.mode:
try:
mode_args = rt.get_option(f'modes/@{options.mode}/options')
# Parse the mode's options and reparse the command-line
options = argparser.parse_args(mode_args)
options = argparser.parse_args(namespace=options.cmd_options)
options.update_config(rt.site_config)
except ConfigError as e:
printer.error('could not obtain execution mode: %s' % e)
sys.exit(1)
if (os_ext.samefile(rt.stage_prefix, rt.output_prefix)
and not site_config.get('general/0/keep_stage_files')):
printer.error("stage and output refer to the same directory; "
"if this is on purpose, please use the "
"'--keep-stage-files' option.")
sys.exit(1)
# Show configuration after everything is set up
if options.show_config_param:
config_param = options.show_config_param
if config_param == 'all':
printer.info(str(rt.site_config))
else:
value = rt.get_option(config_param)
if value is None:
printer.error(
f'no such configuration parameter found: {config_param}')
else:
printer.info(json.dumps(value, indent=2))
sys.exit(0)
# Setup the check loader
loader = RegressionCheckLoader(
load_path=site_config.get('general/0/check_search_path'),
recurse=site_config.get('general/0/check_search_recursive'),
ignore_conflicts=site_config.get('general/0/ignore_check_conflicts'))
printer.debug(argparse.format_options(options))
def print_infoline(param, value):
param = param + ':'
printer.info(f" {param.ljust(18)} {value}")
# Print command line
printer.info(f"[ReFrame Setup]")
print_infoline('version', os_ext.reframe_version())
print_infoline('command', repr(' '.join(sys.argv)))
print_infoline('launched by',
f"{os_ext.osuser() or '<unknown>'}@{socket.gethostname()}")
print_infoline('working directory', repr(os.getcwd()))
print_infoline(
'check search path', f"{'(R)' if loader.recurse else ''} "
f"{':'.join(loader.load_path)!r}")
print_infoline('stage directory', repr(rt.stage_prefix))
print_infoline('output directory', repr(rt.output_prefix))
printer.info('')
try:
# Locate and load checks
try:
checks_found = loader.load_all()
except OSError as e:
raise ReframeError from e
# Filter checks by name
checks_matched = checks_found
if options.exclude_names:
for name in options.exclude_names:
checks_matched = filter(filters.have_not_name(name),
checks_matched)
if options.names:
checks_matched = filter(filters.have_name('|'.join(options.names)),
checks_matched)
# Filter checks by tags
for tag in options.tags:
checks_matched = filter(filters.have_tag(tag), checks_matched)
# Filter checks by prgenv
if not options.skip_prgenv_check:
for prgenv in options.prgenv:
checks_matched = filter(filters.have_prgenv(prgenv),
checks_matched)
# Filter checks by system
if not options.skip_system_check:
checks_matched = filter(
filters.have_partition(rt.system.partitions), checks_matched)
# Filter checks further
if options.gpu_only and options.cpu_only:
printer.error("options `--gpu-only' and `--cpu-only' "
"are mutually exclusive")
sys.exit(1)
if options.gpu_only:
checks_matched = filter(filters.have_gpu_only(), checks_matched)
elif options.cpu_only:
checks_matched = filter(filters.have_cpu_only(), checks_matched)
# Determine the allowed programming environments
allowed_environs = {
e.name
for env_patt in options.prgenv for p in rt.system.partitions
for e in p.environs if re.match(env_patt, e.name)
}
# Generate the test cases, validate dependencies and sort them
checks_matched = list(checks_matched)
testcases = generate_testcases(checks_matched,
options.skip_system_check,
options.skip_prgenv_check,
allowed_environs)
testgraph = dependency.build_deps(testcases)
dependency.validate_deps(testgraph)
testcases = dependency.toposort(testgraph)
# Manipulate ReFrame's environment
if site_config.get('general/0/purge_environment'):
rt.modules_system.unload_all()
else:
for m in site_config.get('general/0/unload_modules'):
rt.modules_system.unload_module(m)
# Load the environment for the current system
try:
runtime.loadenv(rt.system.preload_environ)
except EnvironError as e:
printer.error("failed to load current system's environment; "
"please check your configuration")
printer.debug(str(e))
raise
for m in site_config.get('general/0/user_modules'):
try:
rt.modules_system.load_module(m, force=True)
except EnvironError as e:
printer.warning("could not load module '%s' correctly: "
"Skipping..." % m)
printer.debug(str(e))
if options.flex_alloc_tasks:
printer.warning("`--flex-alloc-tasks' is deprecated and "
"will be removed in the future; "
"you should use --flex-alloc-nodes instead")
options.flex_alloc_nodes = (options.flex_alloc_nodes
or options.flex_alloc_tasks)
options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle'
# Act on checks
success = True
if options.list:
# List matched checks
list_checks(list(checks_matched), printer)
elif options.list_detailed:
# List matched checks with details
list_checks(list(checks_matched), printer, detailed=True)
elif options.run:
# Setup the execution policy
if options.exec_policy == 'serial':
exec_policy = SerialExecutionPolicy()
elif options.exec_policy == 'async':
exec_policy = AsynchronousExecutionPolicy()
else:
# This should not happen, since choices are handled by
# argparser
printer.error("unknown execution policy `%s': Exiting...")
sys.exit(1)
exec_policy.skip_system_check = options.skip_system_check
exec_policy.force_local = options.force_local
exec_policy.strict_check = options.strict
exec_policy.skip_sanity_check = options.skip_sanity_check
exec_policy.skip_performance_check = options.skip_performance_check
exec_policy.keep_stage_files = site_config.get(
'general/0/keep_stage_files')
try:
errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
if sched_flex_alloc_nodes <= 0:
raise ConfigError(errmsg.format(options.flex_alloc_nodes))
except ValueError:
if not options.flex_alloc_nodes.casefold() in {'idle', 'all'}:
raise ConfigError(errmsg.format(
options.flex_alloc_nodes)) from None
sched_flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes
exec_policy.flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_account = options.account
exec_policy.sched_partition = options.partition
exec_policy.sched_reservation = options.reservation
exec_policy.sched_nodelist = options.nodelist
exec_policy.sched_exclude_nodelist = options.exclude_nodes
exec_policy.sched_options = options.job_options
try:
max_retries = int(options.max_retries)
except ValueError:
raise ConfigError('--max-retries is not a valid integer: %s' %
max_retries) from None
runner = Runner(exec_policy, printer, max_retries)
try:
runner.runall(testcases)
finally:
# Print a retry report if we did any retries
if runner.stats.failures(run=0):
printer.info(runner.stats.retry_report())
# Print a failure report if we had failures in the last run
if runner.stats.failures():
printer.info(runner.stats.failure_report())
success = False
if options.failure_stats:
printer.info(runner.stats.failure_stats())
if options.performance_report:
printer.info(runner.stats.performance_report())
else:
printer.error("No action specified. Please specify `-l'/`-L' for "
"listing or `-r' for running. "
"Try `%s -h' for more options." % argparser.prog)
sys.exit(1)
if not success:
sys.exit(1)
sys.exit(0)
except KeyboardInterrupt:
sys.exit(1)
except ReframeError as e:
printer.error(str(e))
sys.exit(1)
except (Exception, ReframeFatalError):
printer.error(format_exception(*sys.exc_info()))
sys.exit(1)
finally:
try:
if site_config.get('general/0/save_log_files'):
logging.save_log_files(rt.output_prefix)
except OSError as e:
printer.error('could not save log file: %s' % e)
sys.exit(1)
def loader_with_path():
return RegressionCheckLoader(
['unittests/resources/checks', 'unittests/foobar'],
ignore_conflicts=True)
def loader():
return RegressionCheckLoader(['.'], ignore_conflicts=True)
def main():
# Setup command line options
argparser = argparse.ArgumentParser()
output_options = argparser.add_argument_group(
'Options controlling ReFrame output')
locate_options = argparser.add_argument_group(
'Options for discovering checks')
select_options = argparser.add_argument_group(
'Options for selecting checks')
action_options = argparser.add_argument_group(
'Options controlling actions')
run_options = argparser.add_argument_group(
'Options controlling the execution of checks')
env_options = argparser.add_argument_group(
'Options controlling the ReFrame environment')
misc_options = argparser.add_argument_group('Miscellaneous options')
# Output directory options
output_options.add_argument('--prefix',
action='store',
metavar='DIR',
help='Set general directory prefix to DIR',
envvar='RFM_PREFIX',
configvar='systems/prefix')
output_options.add_argument('-o',
'--output',
action='store',
metavar='DIR',
help='Set output directory prefix to DIR',
envvar='RFM_OUTPUT_DIR',
configvar='systems/outputdir')
output_options.add_argument('-s',
'--stage',
action='store',
metavar='DIR',
help='Set stage directory prefix to DIR',
envvar='RFM_STAGE_DIR',
configvar='systems/stagedir')
output_options.add_argument(
'--timestamp',
action='store',
nargs='?',
const='',
metavar='TIMEFMT',
help=('Append a timestamp to the output and stage directory prefixes '
'(default: "%%FT%%T")'),
envvar='RFM_TIMESTAMP_DIRS',
configvar='general/timestamp_dirs')
output_options.add_argument(
'--perflogdir',
action='store',
metavar='DIR',
help=('Set performance log data directory prefix '
'(relevant only to the filelog log handler)'),
envvar='RFM_PERFLOG_DIR',
configvar='logging/handlers_perflog/filelog_basedir')
output_options.add_argument(
'--keep-stage-files',
action='store_true',
help='Keep stage directories even for successful checks',
envvar='RFM_KEEP_STAGE_FILES',
configvar='general/keep_stage_files')
output_options.add_argument('--dont-restage',
action='store_false',
dest='clean_stagedir',
help='Reuse the test stage directory',
envvar='RFM_CLEAN_STAGEDIR',
configvar='general/clean_stagedir')
output_options.add_argument(
'--save-log-files',
action='store_true',
default=False,
help='Save ReFrame log files to the output directory',
envvar='RFM_SAVE_LOG_FILES',
configvar='general/save_log_files')
output_options.add_argument('--report-file',
action='store',
metavar='FILE',
help="Store JSON run report in FILE",
envvar='RFM_REPORT_FILE',
configvar='general/report_file')
# Check discovery options
locate_options.add_argument('-c',
'--checkpath',
action='append',
metavar='PATH',
help="Add PATH to the check search path list",
envvar='RFM_CHECK_SEARCH_PATH :',
configvar='general/check_search_path')
locate_options.add_argument(
'-R',
'--recursive',
action='store_true',
help='Search for checks in the search path recursively',
envvar='RFM_CHECK_SEARCH_RECURSIVE',
configvar='general/check_search_recursive')
locate_options.add_argument('--ignore-check-conflicts',
action='store_true',
help='Skip checks with conflicting names',
envvar='RFM_IGNORE_CHECK_CONFLICTS',
configvar='general/ignore_check_conflicts')
# Select options
select_options.add_argument(
'-t',
'--tag',
action='append',
dest='tags',
metavar='PATTERN',
default=[],
help='Select checks with at least one tag matching PATTERN')
select_options.add_argument(
'-n',
'--name',
action='append',
dest='names',
default=[],
metavar='PATTERN',
help='Select checks whose name matches PATTERN')
select_options.add_argument(
'-x',
'--exclude',
action='append',
dest='exclude_names',
metavar='PATTERN',
default=[],
help='Exclude checks whose name matches PATTERN')
select_options.add_argument(
'-p',
'--prgenv',
action='append',
default=[r'.*'],
metavar='PATTERN',
help=('Select checks with at least one '
'programming environment matching PATTERN'))
select_options.add_argument('--gpu-only',
action='store_true',
help='Select only GPU checks')
select_options.add_argument('--cpu-only',
action='store_true',
help='Select only CPU checks')
# Action options
action_options.add_argument('-l',
'--list',
action='store_true',
help='List the selected checks')
action_options.add_argument(
'-L',
'--list-detailed',
action='store_true',
help='List the selected checks providing details for each test')
action_options.add_argument('-r',
'--run',
action='store_true',
help='Run the selected checks')
# Run options
run_options.add_argument('-J',
'--job-option',
action='append',
metavar='OPT',
dest='job_options',
default=[],
help='Pass option OPT to job scheduler')
run_options.add_argument('--force-local',
action='store_true',
help='Force local execution of checks')
run_options.add_argument('--skip-sanity-check',
action='store_true',
help='Skip sanity checking')
run_options.add_argument('--skip-performance-check',
action='store_true',
help='Skip performance checking')
run_options.add_argument('--strict',
action='store_true',
help='Enforce strict performance checking')
run_options.add_argument('--skip-system-check',
action='store_true',
help='Skip system check')
run_options.add_argument('--skip-prgenv-check',
action='store_true',
help='Skip programming environment check')
run_options.add_argument(
'--exec-policy',
metavar='POLICY',
action='store',
choices=['async', 'serial'],
default='async',
help='Set the execution policy of ReFrame (default: "async")')
run_options.add_argument('--mode',
action='store',
help='Execution mode to use')
run_options.add_argument(
'--max-retries',
metavar='NUM',
action='store',
default=0,
help='Set the maximum number of times a failed regression test '
'may be retried (default: 0)')
run_options.add_argument(
'--flex-alloc-nodes',
action='store',
dest='flex_alloc_nodes',
metavar='{all|STATE|NUM}',
default=None,
help='Set strategy for the flexible node allocation (default: "idle").'
)
run_options.add_argument('--disable-hook',
action='append',
metavar='NAME',
dest='hooks',
default=[],
help='Disable a pipeline hook for this run')
env_options.add_argument('-M',
'--map-module',
action='append',
metavar='MAPPING',
dest='module_mappings',
default=[],
help='Add a module mapping',
envvar='RFM_MODULE_MAPPINGS ,',
configvar='general/module_mappings')
env_options.add_argument(
'-m',
'--module',
action='append',
default=[],
metavar='MOD',
dest='user_modules',
help='Load module MOD before running any regression check',
envvar='RFM_USER_MODULES ,',
configvar='general/user_modules')
env_options.add_argument('--module-mappings',
action='store',
metavar='FILE',
dest='module_map_file',
help='Load module mappings from FILE',
envvar='RFM_MODULE_MAP_FILE',
configvar='general/module_map_file')
env_options.add_argument(
'-u',
'--unload-module',
action='append',
metavar='MOD',
dest='unload_modules',
default=[],
help='Unload module MOD before running any regression check',
envvar='RFM_UNLOAD_MODULES ,',
configvar='general/unload_modules')
env_options.add_argument(
'--purge-env',
action='store_true',
dest='purge_env',
default=False,
help='Unload all modules before running any regression check',
envvar='RFM_PURGE_ENVIRONMENT',
configvar='general/purge_environment')
env_options.add_argument(
'--non-default-craype',
action='store_true',
help='Test a non-default Cray Programming Environment',
envvar='RFM_NON_DEFAULT_CRAYPE',
configvar='general/non_default_craype')
# Miscellaneous options
misc_options.add_argument('-C',
'--config-file',
action='store',
dest='config_file',
metavar='FILE',
help='Set configuration file',
envvar='RFM_CONFIG_FILE')
misc_options.add_argument('--nocolor',
action='store_false',
dest='colorize',
help='Disable coloring of output',
envvar='RFM_COLORIZE',
configvar='general/colorize')
misc_options.add_argument('--failure-stats',
action='store_true',
help='Print failure statistics')
misc_options.add_argument('--performance-report',
action='store_true',
help='Print a report for performance tests')
misc_options.add_argument(
'--show-config',
action='store',
nargs='?',
const='all',
metavar='PARAM',
help='Print the value of configuration parameter PARAM and exit')
misc_options.add_argument('--system',
action='store',
help='Load configuration for SYSTEM',
envvar='RFM_SYSTEM')
misc_options.add_argument(
'--upgrade-config-file',
action='store',
metavar='OLD[:NEW]',
help='Upgrade ReFrame 2.x configuration file to ReFrame 3.x syntax')
misc_options.add_argument('-V',
'--version',
action='version',
version=os_ext.reframe_version())
misc_options.add_argument('-v',
'--verbose',
action='count',
help='Increase verbosity level of output',
envvar='RFM_VERBOSE',
configvar='general/verbose')
# Options not associated with command-line arguments
argparser.add_argument(
dest='graylog_server',
envvar='RFM_GRAYLOG_ADDRESS',
configvar='logging/handlers_perflog/graylog_address',
help='Graylog server address')
argparser.add_argument(dest='syslog_address',
envvar='RFM_SYSLOG_ADDRESS',
configvar='logging/handlers_perflog/syslog_address',
help='Syslog server address')
argparser.add_argument(dest='ignore_reqnodenotavail',
envvar='RFM_IGNORE_REQNODENOTAVAIL',
configvar='schedulers/ignore_reqnodenotavail',
action='store_true',
help='Graylog server address')
argparser.add_argument(dest='use_login_shell',
envvar='RFM_USE_LOGIN_SHELL',
configvar='general/use_login_shell',
action='store_true',
help='Use a login shell for job scripts')
if len(sys.argv) == 1:
argparser.print_help()
sys.exit(1)
# Parse command line
options = argparser.parse_args()
# First configure logging with our generic configuration so as to be able
# to print pretty messages; logging will be reconfigured by user's
# configuration later
site_config = config.load_config(
os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py'))
site_config.select_subconfig('generic')
options.update_config(site_config)
logging.configure_logging(site_config)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer = PrettyPrinter()
printer.colorize = site_config.get('general/0/colorize')
printer.inc_verbosity(site_config.get('general/0/verbose'))
if os.getenv('RFM_GRAYLOG_SERVER'):
printer.warning(
'RFM_GRAYLOG_SERVER environment variable is deprecated; '
'please use RFM_GRAYLOG_ADDRESS instead')
os.environ['RFM_GRAYLOG_ADDRESS'] = os.getenv('RFM_GRAYLOG_SERVER')
if options.upgrade_config_file is not None:
old_config, *new_config = options.upgrade_config_file.split(':',
maxsplit=1)
new_config = new_config[0] if new_config else None
try:
new_config = config.convert_old_config(old_config, new_config)
except Exception as e:
printer.error(f'could not convert file: {e}')
sys.exit(1)
printer.info(f'Conversion successful! '
f'The converted file can be found at {new_config!r}.')
sys.exit(0)
# Now configure ReFrame according to the user configuration file
try:
try:
site_config = config.load_config(options.config_file)
except ReframeDeprecationWarning as e:
printer.warning(e)
converted = config.convert_old_config(options.config_file)
printer.warning(f"configuration file has been converted "
f"to the new syntax here: '{converted}'")
site_config = config.load_config(converted)
site_config.validate()
# We ignore errors about unresolved sections or configuration
# parameters here, because they might be defined at the individual
# partition level and will be caught when we will instantiating
# internally the system and partitions later on.
site_config.select_subconfig(options.system,
ignore_resolve_errors=True)
for err in options.update_config(site_config):
printer.warning(str(err))
# Update options from the selected execution mode
if options.mode:
mode_args = site_config.get(f'modes/@{options.mode}/options')
# Parse the mode's options and reparse the command-line
options = argparser.parse_args(mode_args)
options = argparser.parse_args(namespace=options.cmd_options)
options.update_config(site_config)
logging.configure_logging(site_config)
except (OSError, ConfigError) as e:
printer.error(f'failed to load configuration: {e}')
sys.exit(1)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer.colorize = site_config.get('general/0/colorize')
printer.inc_verbosity(site_config.get('general/0/verbose'))
try:
runtime.init_runtime(site_config)
except ConfigError as e:
printer.error(f'failed to initialize runtime: {e}')
sys.exit(1)
rt = runtime.runtime()
try:
if site_config.get('general/0/module_map_file'):
rt.modules_system.load_mapping_from_file(
site_config.get('general/0/module_map_file'))
if site_config.get('general/0/module_mappings'):
for m in site_config.get('general/0/module_mappings'):
rt.modules_system.load_mapping(m)
except (ConfigError, OSError) as e:
printer.error('could not load module mappings: %s' % e)
sys.exit(1)
if (os_ext.samefile(rt.stage_prefix, rt.output_prefix)
and not site_config.get('general/0/keep_stage_files')):
printer.error("stage and output refer to the same directory; "
"if this is on purpose, please use the "
"'--keep-stage-files' option.")
sys.exit(1)
# Show configuration after everything is set up
if options.show_config:
config_param = options.show_config
if config_param == 'all':
printer.info(str(rt.site_config))
else:
value = rt.get_option(config_param)
if value is None:
printer.error(
f'no such configuration parameter found: {config_param}')
else:
printer.info(json.dumps(value, indent=2))
sys.exit(0)
printer.debug(format_env(options.env_vars))
# Setup the check loader
loader = RegressionCheckLoader(
load_path=site_config.get('general/0/check_search_path'),
recurse=site_config.get('general/0/check_search_recursive'),
ignore_conflicts=site_config.get('general/0/ignore_check_conflicts'))
def print_infoline(param, value):
param = param + ':'
printer.info(f" {param.ljust(18)} {value}")
session_info = {
'cmdline': ' '.join(sys.argv),
'config_file': rt.site_config.filename,
'data_version': '1.0',
'hostname': socket.gethostname(),
'prefix_output': rt.output_prefix,
'prefix_stage': rt.stage_prefix,
'user': os_ext.osuser(),
'version': os_ext.reframe_version(),
'workdir': os.getcwd(),
}
# Print command line
printer.info(f"[ReFrame Setup]")
print_infoline('version', session_info['version'])
print_infoline('command', repr(session_info['cmdline']))
print_infoline(
f"launched by",
f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}")
print_infoline('working directory', repr(session_info['workdir']))
print_infoline('settings file', f"{session_info['config_file']!r}")
print_infoline(
'check search path', f"{'(R) ' if loader.recurse else ''}"
f"{':'.join(loader.load_path)!r}")
print_infoline('stage directory', repr(session_info['prefix_stage']))
print_infoline('output directory', repr(session_info['prefix_output']))
printer.info('')
try:
# Locate and load checks
try:
checks_found = loader.load_all()
except OSError as e:
raise ReframeError from e
# Filter checks by name
checks_matched = checks_found
if options.exclude_names:
for name in options.exclude_names:
checks_matched = filter(filters.have_not_name(name),
checks_matched)
if options.names:
checks_matched = filter(filters.have_name('|'.join(options.names)),
checks_matched)
# Filter checks by tags
for tag in options.tags:
checks_matched = filter(filters.have_tag(tag), checks_matched)
# Filter checks by prgenv
if not options.skip_prgenv_check:
for prgenv in options.prgenv:
checks_matched = filter(filters.have_prgenv(prgenv),
checks_matched)
# Filter checks by system
if not options.skip_system_check:
checks_matched = filter(
filters.have_partition(rt.system.partitions), checks_matched)
# Filter checks further
if options.gpu_only and options.cpu_only:
printer.error("options `--gpu-only' and `--cpu-only' "
"are mutually exclusive")
sys.exit(1)
if options.gpu_only:
checks_matched = filter(filters.have_gpu_only(), checks_matched)
elif options.cpu_only:
checks_matched = filter(filters.have_cpu_only(), checks_matched)
# Determine the allowed programming environments
allowed_environs = {
e.name
for env_patt in options.prgenv for p in rt.system.partitions
for e in p.environs if re.match(env_patt, e.name)
}
# Generate the test cases, validate dependencies and sort them
checks_matched = list(checks_matched)
# Disable hooks
for c in checks_matched:
for h in options.hooks:
type(c).disable_hook(h)
testcases = generate_testcases(checks_matched,
options.skip_system_check,
options.skip_prgenv_check,
allowed_environs)
testgraph = dependency.build_deps(testcases)
dependency.validate_deps(testgraph)
testcases = dependency.toposort(testgraph)
# Manipulate ReFrame's environment
if site_config.get('general/0/purge_environment'):
rt.modules_system.unload_all()
else:
for m in site_config.get('general/0/unload_modules'):
rt.modules_system.unload_module(m)
# Load the environment for the current system
try:
runtime.loadenv(rt.system.preload_environ)
except EnvironError as e:
printer.error("failed to load current system's environment; "
"please check your configuration")
printer.debug(str(e))
raise
for m in site_config.get('general/0/user_modules'):
try:
rt.modules_system.load_module(m, force=True)
except EnvironError as e:
printer.warning("could not load module '%s' correctly: "
"Skipping..." % m)
printer.debug(str(e))
options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle'
# Act on checks
success = True
if options.list or options.list_detailed:
list_checks(list(checks_matched), printer, options.list_detailed)
elif options.run:
# Setup the execution policy
if options.exec_policy == 'serial':
exec_policy = SerialExecutionPolicy()
elif options.exec_policy == 'async':
exec_policy = AsynchronousExecutionPolicy()
else:
# This should not happen, since choices are handled by
# argparser
printer.error("unknown execution policy `%s': Exiting...")
sys.exit(1)
exec_policy.skip_system_check = options.skip_system_check
exec_policy.force_local = options.force_local
exec_policy.strict_check = options.strict
exec_policy.skip_sanity_check = options.skip_sanity_check
exec_policy.skip_performance_check = options.skip_performance_check
exec_policy.keep_stage_files = site_config.get(
'general/0/keep_stage_files')
try:
errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
if sched_flex_alloc_nodes <= 0:
raise ConfigError(errmsg.format(options.flex_alloc_nodes))
except ValueError:
sched_flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes
parsed_job_options = []
for opt in options.job_options:
if opt.startswith('-') or opt.startswith('#'):
parsed_job_options.append(opt)
elif len(opt) == 1:
parsed_job_options.append(f'-{opt}')
else:
parsed_job_options.append(f'--{opt}')
exec_policy.sched_options = parsed_job_options
try:
max_retries = int(options.max_retries)
except ValueError:
raise ConfigError('--max-retries is not a valid integer: %s' %
max_retries) from None
runner = Runner(exec_policy, printer, max_retries)
try:
time_start = time.time()
session_info['time_start'] = time.strftime(
'%FT%T%z',
time.localtime(time_start),
)
runner.runall(testcases)
finally:
time_end = time.time()
session_info['time_end'] = time.strftime(
'%FT%T%z', time.localtime(time_end))
session_info['time_elapsed'] = time_end - time_start
# Print a retry report if we did any retries
if runner.stats.failures(run=0):
printer.info(runner.stats.retry_report())
# Print a failure report if we had failures in the last run
if runner.stats.failures():
printer.info(runner.stats.failure_report())
success = False
if options.failure_stats:
printer.info(runner.stats.failure_stats())
if options.performance_report:
printer.info(runner.stats.performance_report())
# Generate the report for this session
report_file = os.path.normpath(
os_ext.expandvars(rt.get_option('general/0/report_file')))
basedir = os.path.dirname(report_file)
if basedir:
os.makedirs(basedir, exist_ok=True)
# Build final JSON report
run_stats = runner.stats.json()
session_info.update({
'num_cases':
run_stats[0]['num_cases'],
'num_failures':
run_stats[-1]['num_failures']
})
json_report = {'session_info': session_info, 'runs': run_stats}
report_file = generate_report_filename(report_file)
try:
with open(report_file, 'w') as fp:
jsonext.dump(json_report, fp, indent=2)
except OSError as e:
printer.warning(
f'failed to generate report in {report_file!r}: {e}')
else:
printer.error("No action specified. Please specify `-l'/`-L' for "
"listing or `-r' for running. "
"Try `%s -h' for more options." % argparser.prog)
sys.exit(1)
if not success:
sys.exit(1)
sys.exit(0)
except KeyboardInterrupt:
sys.exit(1)
except ReframeError as e:
printer.error(str(e))
sys.exit(1)
except (Exception, ReframeFatalError):
printer.error(format_exception(*sys.exc_info()))
sys.exit(1)
finally:
try:
log_files = logging.log_files()
if site_config.get('general/0/save_log_files'):
log_files = logging.save_log_files(rt.output_prefix)
except OSError as e:
printer.error('could not save log file: %s' % e)
sys.exit(1)
finally:
if not log_files:
msg = '<no log file was generated>'
else:
msg = f'{", ".join(repr(f) for f in log_files)}'
printer.info(f'Log file(s) saved in: {msg}')
def loader():
return RegressionCheckLoader([
'unittests/resources/checks_unlisted/deps_simple.py'
])
def main():
# Setup command line options
argparser = argparse.ArgumentParser()
output_options = argparser.add_argument_group('Options controlling output')
locate_options = argparser.add_argument_group(
'Options for locating checks')
select_options = argparser.add_argument_group(
'Options for selecting checks')
action_options = argparser.add_argument_group(
'Options controlling actions')
run_options = argparser.add_argument_group(
'Options controlling execution of checks')
env_options = argparser.add_argument_group(
'Options controlling environment')
misc_options = argparser.add_argument_group('Miscellaneous options')
# Output directory options
output_options.add_argument('--prefix',
action='store',
metavar='DIR',
help='Set output directory prefix to DIR')
output_options.add_argument('-o',
'--output',
action='store',
metavar='DIR',
help='Set output directory to DIR')
output_options.add_argument('-s',
'--stage',
action='store',
metavar='DIR',
help='Set stage directory to DIR')
output_options.add_argument(
'--perflogdir',
action='store',
metavar='DIR',
help='Set directory prefix for the performance logs '
'(default: ${prefix}/perflogs, '
'relevant only if the filelog backend is used)')
output_options.add_argument(
'--keep-stage-files',
action='store_true',
help='Keep stage directory even if check is successful')
output_options.add_argument(
'--save-log-files',
action='store_true',
default=False,
help='Copy the log file from the work dir to the output dir at the '
'end of the program')
# Check discovery options
locate_options.add_argument('-c',
'--checkpath',
action='append',
metavar='DIR|FILE',
help='Search for checks in DIR or FILE')
locate_options.add_argument('-R',
'--recursive',
action='store_true',
help='Load checks recursively')
locate_options.add_argument('--ignore-check-conflicts',
action='store_true',
help='Skip checks with conflicting names')
# Select options
select_options.add_argument('-t',
'--tag',
action='append',
dest='tags',
default=[],
help='Select checks matching TAG')
select_options.add_argument('-n',
'--name',
action='append',
dest='names',
default=[],
metavar='NAME',
help='Select checks with NAME')
select_options.add_argument('-x',
'--exclude',
action='append',
dest='exclude_names',
metavar='NAME',
default=[],
help='Exclude checks with NAME')
select_options.add_argument(
'-p',
'--prgenv',
action='append',
default=[r'.*'],
help='Select tests for PRGENV programming environment only')
select_options.add_argument('--gpu-only',
action='store_true',
help='Select only GPU tests')
select_options.add_argument('--cpu-only',
action='store_true',
help='Select only CPU tests')
# Action options
action_options.add_argument('-l',
'--list',
action='store_true',
help='List matched regression checks')
action_options.add_argument(
'-L',
'--list-detailed',
action='store_true',
help='List matched regression checks with a detailed description')
action_options.add_argument('-r',
'--run',
action='store_true',
help='Run regression with the selected checks')
# Run options
run_options.add_argument('-A',
'--account',
action='store',
help='Use ACCOUNT for submitting jobs')
run_options.add_argument('-P',
'--partition',
action='store',
metavar='PART',
help='Use PART for submitting jobs')
run_options.add_argument('--reservation',
action='store',
metavar='RES',
help='Use RES for submitting jobs')
run_options.add_argument('--nodelist',
action='store',
help='Run checks on the selected list of nodes')
run_options.add_argument(
'--exclude-nodes',
action='store',
metavar='NODELIST',
help='Exclude the list of nodes from running checks')
run_options.add_argument('--job-option',
action='append',
metavar='OPT',
dest='job_options',
default=[],
help='Pass OPT to job scheduler')
run_options.add_argument('--force-local',
action='store_true',
help='Force local execution of checks')
run_options.add_argument('--skip-sanity-check',
action='store_true',
help='Skip sanity checking')
run_options.add_argument('--skip-performance-check',
action='store_true',
help='Skip performance checking')
run_options.add_argument('--strict',
action='store_true',
help='Force strict performance checking')
run_options.add_argument('--skip-system-check',
action='store_true',
help='Skip system check')
run_options.add_argument('--skip-prgenv-check',
action='store_true',
help='Skip prog. environment check')
run_options.add_argument(
'--exec-policy',
metavar='POLICY',
action='store',
choices=['serial', 'async'],
default='serial',
help='Specify the execution policy for running the regression tests. '
'Available policies: "serial" (default), "async"')
run_options.add_argument('--mode',
action='store',
help='Execution mode to use')
run_options.add_argument(
'--max-retries',
metavar='NUM',
action='store',
default=0,
help='Specify the maximum number of times a failed regression test '
'may be retried (default: 0)')
run_options.add_argument(
'--flex-alloc-tasks',
action='store',
dest='flex_alloc_tasks',
metavar='{all|idle|NUM}',
default='idle',
help="Strategy for flexible task allocation (default: 'idle').")
env_options.add_argument('-M',
'--map-module',
action='append',
metavar='MAPPING',
dest='module_mappings',
default=[],
help='Apply a single module mapping')
env_options.add_argument(
'-m',
'--module',
action='append',
default=[],
metavar='MOD',
dest='user_modules',
help='Load module MOD before running the regression suite')
env_options.add_argument('--module-mappings',
action='store',
metavar='FILE',
dest='module_map_file',
help='Apply module mappings defined in FILE')
env_options.add_argument(
'-u',
'--unload-module',
action='append',
metavar='MOD',
dest='unload_modules',
default=[],
help='Unload module MOD before running the regression suite')
env_options.add_argument(
'--purge-env',
action='store_true',
dest='purge_env',
default=False,
help='Purge environment before running the regression suite')
# Miscellaneous options
misc_options.add_argument(
'-C',
'--config-file',
action='store',
dest='config_file',
metavar='FILE',
default=os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py'),
help='Specify a custom config-file for the machine. '
'(default: %s' %
os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py'))
misc_options.add_argument('--nocolor',
action='store_false',
dest='colorize',
default=True,
help='Disable coloring of output')
misc_options.add_argument('--performance-report',
action='store_true',
help='Print the performance report')
# FIXME: This should move to env_options as soon as
# https://github.com/eth-cscs/reframe/pull/946 is merged
misc_options.add_argument('--non-default-craype',
action='store_true',
default=False,
help='Test a non-default Cray PE')
misc_options.add_argument(
'--show-config',
action='store_true',
help='Print configuration of the current system and exit')
misc_options.add_argument(
'--show-config-env',
action='store',
metavar='ENV',
help='Print configuration of environment ENV and exit')
misc_options.add_argument('--system',
action='store',
help='Load SYSTEM configuration explicitly')
misc_options.add_argument(
'--timestamp',
action='store',
nargs='?',
const='%FT%T',
metavar='TIMEFMT',
help='Append a timestamp component to the regression directories'
'(default format "%%FT%%T")')
misc_options.add_argument('-V',
'--version',
action='version',
version=reframe.VERSION)
misc_options.add_argument('-v',
'--verbose',
action='count',
default=0,
help='Increase verbosity level of output')
if len(sys.argv) == 1:
argparser.print_help()
sys.exit(1)
# Parse command line
options = argparser.parse_args()
# Load configuration
try:
settings = config.load_settings_from_file(options.config_file)
except (OSError, ReframeError) as e:
sys.stderr.write('%s: could not load settings: %s\n' %
(sys.argv[0], e))
sys.exit(1)
# Configure logging
try:
logging.configure_logging(settings.logging_config),
except (OSError, ConfigError) as e:
sys.stderr.write('could not configure logging: %s\n' % e)
sys.exit(1)
# Set colors in logger
logging.getlogger().colorize = options.colorize
# Setup printer
printer = PrettyPrinter()
printer.colorize = options.colorize
if options.verbose:
printer.inc_verbosity(options.verbose)
try:
runtime.init_runtime(settings.site_configuration,
options.system,
non_default_craype=options.non_default_craype)
except SystemAutodetectionError:
printer.warning(
'could not find a configuration entry for the current system; '
'falling back to a generic system configuration; '
'please check the online documentation on how to configure '
'ReFrame for your system.')
settings.site_configuration['systems'] = {
'generic': {
'descr': 'Generic fallback system configuration',
'hostnames': ['localhost'],
'partitions': {
'login': {
'scheduler': 'local',
'environs': ['builtin-gcc'],
'descr': 'Login nodes'
}
}
}
}
settings.site_configuration['environments'] = {
'*': {
'builtin-gcc': {
'type': 'ProgEnvironment',
'cc': 'gcc',
'cxx': 'g++',
'ftn': 'gfortran',
}
}
}
runtime.init_runtime(settings.site_configuration,
'generic',
non_default_craype=options.non_default_craype)
except Exception as e:
printer.error('configuration error: %s' % e)
printer.verbose(''.join(traceback.format_exception(*sys.exc_info())))
sys.exit(1)
rt = runtime.runtime()
try:
if options.module_map_file:
rt.modules_system.load_mapping_from_file(options.module_map_file)
if options.module_mappings:
for m in options.module_mappings:
rt.modules_system.load_mapping(m)
except (ConfigError, OSError) as e:
printer.error('could not load module mappings: %s' % e)
sys.exit(1)
if options.mode:
try:
mode_args = rt.mode(options.mode)
# Parse the mode's options and reparse the command-line
options = argparser.parse_args(mode_args)
options = argparser.parse_args(namespace=options)
except ConfigError as e:
printer.error('could not obtain execution mode: %s' % e)
sys.exit(1)
# Adjust system directories
if options.prefix:
# if prefix is set, reset all other directories
rt.resources.prefix = os_ext.expandvars(options.prefix)
rt.resources.outputdir = None
rt.resources.stagedir = None
if options.output:
rt.resources.outputdir = os_ext.expandvars(options.output)
if options.stage:
rt.resources.stagedir = os_ext.expandvars(options.stage)
if (os_ext.samefile(rt.resources.stage_prefix, rt.resources.output_prefix)
and not options.keep_stage_files):
printer.error('stage and output refer to the same directory; '
'if this is on purpose, please use also the '
"`--keep-stage-files' option.")
sys.exit(1)
if options.timestamp:
rt.resources.timefmt = options.timestamp
# Configure performance logging
# NOTE: we need resources to be configured in order to set the global
# perf. logging prefix correctly
if options.perflogdir:
rt.resources.perflogdir = os_ext.expandvars(options.perflogdir)
logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = (
rt.resources.perflog_prefix)
# Show configuration after everything is set up
if options.show_config:
printer.info(rt.show_config())
sys.exit(0)
if options.show_config_env:
envname = options.show_config_env
for p in rt.system.partitions:
environ = p.environment(envname)
if environ:
break
if environ is None:
printer.error('no such environment: ' + envname)
sys.exit(1)
printer.info(environ.details())
sys.exit(0)
if hasattr(settings, 'perf_logging_config'):
try:
logging.configure_perflogging(settings.perf_logging_config)
except (OSError, ConfigError) as e:
printer.error('could not configure performance logging: %s\n' % e)
sys.exit(1)
else:
printer.warning('no performance logging is configured; '
'please check documentation')
# Setup the check loader
if options.checkpath:
load_path = []
for d in options.checkpath:
d = os_ext.expandvars(d)
if not os.path.exists(d):
printer.warning("%s: path `%s' does not exist. Skipping..." %
(argparser.prog, d))
continue
load_path.append(d)
loader = RegressionCheckLoader(
load_path,
recurse=options.recursive,
ignore_conflicts=options.ignore_check_conflicts)
else:
loader = RegressionCheckLoader(load_path=settings.checks_path,
prefix=reframe.INSTALL_PREFIX,
recurse=settings.checks_path_recurse)
printer.debug(argparse.format_options(options))
# Print command line
printer.info('Command line: %s' % ' '.join(sys.argv))
printer.info('Reframe version: ' + reframe.VERSION)
printer.info('Launched by user: '******'<unknown>'))
printer.info('Launched on host: ' + socket.gethostname())
# Print important paths
printer.info('Reframe paths')
printer.info('=============')
printer.info(' Check prefix : %s' % loader.prefix)
printer.info(
'%03s Check search path : %s' %
('(R)' if loader.recurse else '', "'%s'" % ':'.join(loader.load_path)))
printer.info(' Stage dir prefix : %s' % rt.resources.stage_prefix)
printer.info(' Output dir prefix : %s' % rt.resources.output_prefix)
printer.info(
' Perf. logging prefix : %s' %
os.path.abspath(logging.LOG_CONFIG_OPTS['handlers.filelog.prefix']))
try:
# Locate and load checks
try:
checks_found = loader.load_all()
except OSError as e:
raise ReframeError from e
# Filter checks by name
checks_matched = checks_found
if options.exclude_names:
for name in options.exclude_names:
checks_matched = filter(filters.have_not_name(name),
checks_matched)
if options.names:
checks_matched = filter(filters.have_name('|'.join(options.names)),
checks_matched)
# Filter checks by tags
for tag in options.tags:
checks_matched = filter(filters.have_tag(tag), checks_matched)
# Filter checks by prgenv
if not options.skip_prgenv_check:
for prgenv in options.prgenv:
checks_matched = filter(filters.have_prgenv(prgenv),
checks_matched)
# Filter checks by system
if not options.skip_system_check:
checks_matched = filter(
filters.have_partition(rt.system.partitions), checks_matched)
# Filter checks further
if options.gpu_only and options.cpu_only:
printer.error("options `--gpu-only' and `--cpu-only' "
"are mutually exclusive")
sys.exit(1)
if options.gpu_only:
checks_matched = filter(filters.have_gpu_only(), checks_matched)
elif options.cpu_only:
checks_matched = filter(filters.have_cpu_only(), checks_matched)
# Determine the allowed programming environments
allowed_environs = {
e.name
for env_patt in options.prgenv for p in rt.system.partitions
for e in p.environs if re.match(env_patt, e.name)
}
# Generate the test cases, validate dependencies and sort them
checks_matched = list(checks_matched)
testcases = generate_testcases(checks_matched,
options.skip_system_check,
options.skip_prgenv_check,
allowed_environs)
testgraph = dependency.build_deps(testcases)
dependency.validate_deps(testgraph)
testcases = dependency.toposort(testgraph)
# Unload regression's module and load user-specified modules
if hasattr(settings, 'reframe_module'):
printer.warning(
"the 'reframe_module' configuration option will be ignored; "
"please use the '-u' or '--unload-module' options")
if options.purge_env:
rt.modules_system.unload_all()
else:
for m in options.unload_modules:
rt.modules_system.unload_module(m)
# Load the environment for the current system
try:
env.load(rt.system.preload_environ)
except EnvironError as e:
printer.error("failed to load current system's environment; "
"please check your configuration")
printer.debug(str(e))
raise
for m in options.user_modules:
try:
rt.modules_system.load_module(m, force=True)
except EnvironError as e:
printer.warning("could not load module '%s' correctly: "
"Skipping..." % m)
printer.debug(str(e))
# Act on checks
success = True
if options.list:
# List matched checks
list_checks(list(checks_matched), printer)
elif options.list_detailed:
# List matched checks with details
list_checks(list(checks_matched), printer, detailed=True)
elif options.run:
# Setup the execution policy
if options.exec_policy == 'serial':
exec_policy = SerialExecutionPolicy()
elif options.exec_policy == 'async':
exec_policy = AsynchronousExecutionPolicy()
else:
# This should not happen, since choices are handled by
# argparser
printer.error("unknown execution policy `%s': Exiting...")
sys.exit(1)
exec_policy.skip_system_check = options.skip_system_check
exec_policy.force_local = options.force_local
exec_policy.strict_check = options.strict
exec_policy.skip_sanity_check = options.skip_sanity_check
exec_policy.skip_performance_check = options.skip_performance_check
exec_policy.keep_stage_files = options.keep_stage_files
try:
errmsg = "invalid option for --flex-alloc-tasks: '{0}'"
sched_flex_alloc_tasks = int(options.flex_alloc_tasks)
if sched_flex_alloc_tasks <= 0:
raise ConfigError(errmsg.format(options.flex_alloc_tasks))
except ValueError:
if not options.flex_alloc_tasks.lower() in {'idle', 'all'}:
raise ConfigError(errmsg.format(
options.flex_alloc_tasks)) from None
sched_flex_alloc_tasks = options.flex_alloc_tasks
exec_policy.sched_flex_alloc_tasks = sched_flex_alloc_tasks
exec_policy.flex_alloc_tasks = options.flex_alloc_tasks
exec_policy.sched_account = options.account
exec_policy.sched_partition = options.partition
exec_policy.sched_reservation = options.reservation
exec_policy.sched_nodelist = options.nodelist
exec_policy.sched_exclude_nodelist = options.exclude_nodes
exec_policy.sched_options = options.job_options
try:
max_retries = int(options.max_retries)
except ValueError:
raise ConfigError('--max-retries is not a valid integer: %s' %
max_retries) from None
runner = Runner(exec_policy, printer, max_retries)
try:
runner.runall(testcases)
finally:
# Print a retry report if we did any retries
if runner.stats.failures(run=0):
printer.info(runner.stats.retry_report())
# Print a failure report if we had failures in the last run
if runner.stats.failures():
printer.info(runner.stats.failure_report())
success = False
if options.performance_report:
printer.info(runner.stats.performance_report())
else:
printer.info('No action specified. Exiting...')
printer.info("Try `%s -h' for a list of available actions." %
argparser.prog)
sys.exit(1)
if not success:
sys.exit(1)
sys.exit(0)
except KeyboardInterrupt:
sys.exit(1)
except ReframeError as e:
printer.error(str(e))
sys.exit(1)
except (Exception, ReframeFatalError):
printer.error(format_exception(*sys.exc_info()))
sys.exit(1)
finally:
try:
if options.save_log_files:
logging.save_log_files(rt.resources.output_prefix)
except OSError as e:
printer.error('could not save log file: %s' % e)
sys.exit(1)
class TestSerialExecutionPolicy(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['unittests/resources/checks'],
ignore_conflicts=True)
# Setup the runner
self.runner = executors.Runner(policies.SerialExecutionPolicy())
self.checks = self.loader.load_all()
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
# Reset current_run
rt.runtime()._current_run = 0
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
def runall(self, checks, sort=False, *args, **kwargs):
cases = executors.generate_testcases(checks, *args, **kwargs)
if sort:
depgraph = dependency.build_deps(cases)
dependency.validate_deps(depgraph)
cases = dependency.toposort(depgraph)
self.runner.runall(cases)
def assertRunall(self):
# Make sure that all cases finished or failed
for t in self.runner.stats.tasks():
assert t.succeeded or t.failed
def _num_failures_stage(self, stage):
stats = self.runner.stats
return len([t for t in stats.failures() if t.failed_stage == stage])
def assert_all_dead(self):
stats = self.runner.stats
for t in self.runner.stats.tasks():
try:
finished = t.check.poll()
except JobNotStartedError:
finished = True
assert finished
def test_runall(self):
self.runall(self.checks)
stats = self.runner.stats
assert 8 == stats.num_cases()
self.assertRunall()
assert 5 == len(stats.failures())
assert 2 == self._num_failures_stage('setup')
assert 1 == self._num_failures_stage('sanity')
assert 1 == self._num_failures_stage('performance')
assert 1 == self._num_failures_stage('cleanup')
def test_runall_skip_system_check(self):
self.runall(self.checks, skip_system_check=True)
stats = self.runner.stats
assert 9 == stats.num_cases()
self.assertRunall()
assert 5 == len(stats.failures())
assert 2 == self._num_failures_stage('setup')
assert 1 == self._num_failures_stage('sanity')
assert 1 == self._num_failures_stage('performance')
assert 1 == self._num_failures_stage('cleanup')
def test_runall_skip_prgenv_check(self):
self.runall(self.checks, skip_environ_check=True)
stats = self.runner.stats
assert 9 == stats.num_cases()
self.assertRunall()
assert 5 == len(stats.failures())
assert 2 == self._num_failures_stage('setup')
assert 1 == self._num_failures_stage('sanity')
assert 1 == self._num_failures_stage('performance')
assert 1 == self._num_failures_stage('cleanup')
def test_runall_skip_sanity_check(self):
self.runner.policy.skip_sanity_check = True
self.runall(self.checks)
stats = self.runner.stats
assert 8 == stats.num_cases()
self.assertRunall()
assert 4 == len(stats.failures())
assert 2 == self._num_failures_stage('setup')
assert 0 == self._num_failures_stage('sanity')
assert 1 == self._num_failures_stage('performance')
assert 1 == self._num_failures_stage('cleanup')
def test_runall_skip_performance_check(self):
self.runner.policy.skip_performance_check = True
self.runall(self.checks)
stats = self.runner.stats
assert 8 == stats.num_cases()
self.assertRunall()
assert 4 == len(stats.failures())
assert 2 == self._num_failures_stage('setup')
assert 1 == self._num_failures_stage('sanity')
assert 0 == self._num_failures_stage('performance')
assert 1 == self._num_failures_stage('cleanup')
def test_strict_performance_check(self):
self.runner.policy.strict_check = True
self.runall(self.checks)
stats = self.runner.stats
assert 8 == stats.num_cases()
self.assertRunall()
assert 6 == len(stats.failures())
assert 2 == self._num_failures_stage('setup')
assert 1 == self._num_failures_stage('sanity')
assert 2 == self._num_failures_stage('performance')
assert 1 == self._num_failures_stage('cleanup')
def test_force_local_execution(self):
self.runner.policy.force_local = True
self.runall([HelloTest()])
self.assertRunall()
stats = self.runner.stats
for t in stats.tasks():
assert t.check.local
def test_kbd_interrupt_within_test(self):
check = KeyboardInterruptCheck()
with pytest.raises(KeyboardInterrupt):
self.runall([check])
stats = self.runner.stats
assert 1 == len(stats.failures())
self.assert_all_dead()
def test_system_exit_within_test(self):
check = SystemExitCheck()
# This should not raise and should not exit
self.runall([check])
stats = self.runner.stats
assert 1 == len(stats.failures())
def test_retries_bad_check(self):
max_retries = 2
checks = [BadSetupCheck(), BadSetupCheckEarly()]
self.runner._max_retries = max_retries
self.runall(checks)
# Ensure that the test was retried #max_retries times and failed.
assert 2 == self.runner.stats.num_cases()
self.assertRunall()
assert max_retries == rt.runtime().current_run
assert 2 == len(self.runner.stats.failures())
# Ensure that the report does not raise any exception.
self.runner.stats.retry_report()
def test_retries_good_check(self):
max_retries = 2
checks = [HelloTest()]
self.runner._max_retries = max_retries
self.runall(checks)
# Ensure that the test passed without retries.
assert 1 == self.runner.stats.num_cases()
self.assertRunall()
assert 0 == rt.runtime().current_run
assert 0 == len(self.runner.stats.failures())
def test_pass_in_retries(self):
max_retries = 3
run_to_pass = 2
# Create a file containing the current_run; Run 0 will set it to 0,
# run 1 to 1 and so on.
with tempfile.NamedTemporaryFile(mode='wt', delete=False) as fp:
fp.write('0\n')
checks = [RetriesCheck(run_to_pass, fp.name)]
self.runner._max_retries = max_retries
self.runall(checks)
# Ensure that the test passed after retries in run #run_to_pass.
assert 1 == self.runner.stats.num_cases()
self.assertRunall()
assert 1 == len(self.runner.stats.failures(run=0))
assert run_to_pass == rt.runtime().current_run
assert 0 == len(self.runner.stats.failures())
os.remove(fp.name)
def test_dependencies(self):
self.loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/deps_complex.py'])
# Setup the runner
self.checks = self.loader.load_all()
self.runall(self.checks, sort=True)
self.assertRunall()
stats = self.runner.stats
assert stats.num_cases(0) == 10
assert len(stats.failures()) == 4
for tf in stats.failures():
check = tf.testcase.check
_, exc_value, _ = tf.exc_info
if check.name == 'T7' or check.name == 'T9':
assert isinstance(exc_value, TaskDependencyError)
# Check that cleanup is executed properly for successful tests as well
for t in stats.tasks():
check = t.testcase.check
if t.failed:
continue
if t.ref_count == 0:
assert os.path.exists(os.path.join(check.outputdir, 'out.txt'))
def test_sigterm(self):
# Wrapper of self.runall which is used from a child process and
# passes any exception, number of cases and failures to the parent
# process
def _runall(checks, ns):
exc = None
try:
self.runall(checks)
except BaseException as e:
exc = e
finally:
ns.exc = exc
ns.num_cases = self.runner.stats.num_cases()
ns.num_failures = len(self.runner.stats.failures())
with multiprocessing.Manager() as manager:
ns = manager.Namespace()
p = multiprocessing.Process(target=_runall,
args=([SleepCheck(20)], ns))
p.start()
# Allow some time so that the SleepCheck is submitted.
# The sleep time of the submitted test is much larger to
# ensure that it does not finish before the termination signal
time.sleep(0.2)
p.terminate()
p.join()
# Either the test is submitted and it fails due to the termination
# or it is not yet submitted when the termination signal is sent
assert (ns.num_cases, ns.num_failures) in {(1, 1), (0, 0)}
with pytest.raises(ReframeForceExitError,
match='received TERM signal'):
if ns.exc:
raise ns.exc
def test_dependencies_with_retries(self):
self.runner._max_retries = 2
self.test_dependencies()
class TestSerialExecutionPolicy(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['unittests/resources/checks'],
ignore_conflicts=True)
# Setup the runner
self.runner = executors.Runner(policies.SerialExecutionPolicy())
self.checks = self.loader.load_all()
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
# Reset current_run
rt.runtime()._current_run = 0
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
def runall(self, checks, sort=False, *args, **kwargs):
cases = executors.generate_testcases(checks, *args, **kwargs)
if sort:
depgraph = dependency.build_deps(cases)
dependency.validate_deps(depgraph)
cases = dependency.toposort(depgraph)
self.runner.runall(cases)
def _num_failures_stage(self, stage):
stats = self.runner.stats
return len([t for t in stats.failures() if t.failed_stage == stage])
def assert_all_dead(self):
stats = self.runner.stats
for t in self.runner.stats.tasks():
try:
finished = t.check.poll()
except JobNotStartedError:
finished = True
self.assertTrue(finished)
def test_runall(self):
self.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(4, len(stats.failures()))
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_system_check(self):
self.runall(self.checks, skip_system_check=True)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(4, len(stats.failures()))
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_prgenv_check(self):
self.runall(self.checks, skip_environ_check=True)
stats = self.runner.stats
self.assertEqual(8, stats.num_cases())
self.assertEqual(4, len(stats.failures()))
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_sanity_check(self):
self.runner.policy.skip_sanity_check = True
self.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(3, len(stats.failures()))
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(0, self._num_failures_stage('sanity'))
self.assertEqual(1, self._num_failures_stage('performance'))
def test_runall_skip_performance_check(self):
self.runner.policy.skip_performance_check = True
self.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(3, len(stats.failures()))
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(0, self._num_failures_stage('performance'))
def test_strict_performance_check(self):
self.runner.policy.strict_check = True
self.runall(self.checks)
stats = self.runner.stats
self.assertEqual(7, stats.num_cases())
self.assertEqual(5, len(stats.failures()))
self.assertEqual(2, self._num_failures_stage('setup'))
self.assertEqual(1, self._num_failures_stage('sanity'))
self.assertEqual(2, self._num_failures_stage('performance'))
def test_force_local_execution(self):
self.runner.policy.force_local = True
self.runall([HelloTest()])
stats = self.runner.stats
for t in stats.tasks():
self.assertTrue(t.check.local)
def test_kbd_interrupt_within_test(self):
check = KeyboardInterruptCheck()
self.assertRaises(KeyboardInterrupt, self.runall, [check])
stats = self.runner.stats
self.assertEqual(1, len(stats.failures()))
self.assert_all_dead()
def test_system_exit_within_test(self):
check = SystemExitCheck()
# This should not raise and should not exit
self.runall([check])
stats = self.runner.stats
self.assertEqual(1, len(stats.failures()))
def test_retries_bad_check(self):
max_retries = 2
checks = [BadSetupCheck(), BadSetupCheckEarly()]
self.runner._max_retries = max_retries
self.runall(checks)
# Ensure that the test was retried #max_retries times and failed.
self.assertEqual(2, self.runner.stats.num_cases())
self.assertEqual(max_retries, rt.runtime().current_run)
self.assertEqual(2, len(self.runner.stats.failures()))
# Ensure that the report does not raise any exception.
self.runner.stats.retry_report()
def test_retries_good_check(self):
max_retries = 2
checks = [HelloTest()]
self.runner._max_retries = max_retries
self.runall(checks)
# Ensure that the test passed without retries.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(0, rt.runtime().current_run)
self.assertEqual(0, len(self.runner.stats.failures()))
def test_pass_in_retries(self):
max_retries = 3
run_to_pass = 2
# Create a file containing the current_run; Run 0 will set it to 0,
# run 1 to 1 and so on.
with tempfile.NamedTemporaryFile(mode='wt', delete=False) as fp:
fp.write('0\n')
checks = [RetriesCheck(run_to_pass, fp.name)]
self.runner._max_retries = max_retries
self.runall(checks)
# Ensure that the test passed after retries in run #run_to_pass.
self.assertEqual(1, self.runner.stats.num_cases())
self.assertEqual(1, len(self.runner.stats.failures(run=0)))
self.assertEqual(run_to_pass, rt.runtime().current_run)
self.assertEqual(0, len(self.runner.stats.failures()))
os.remove(fp.name)
def test_dependencies(self):
self.loader = RegressionCheckLoader(
['unittests/resources/checks_unlisted/deps_complex.py'])
# Setup the runner
self.checks = self.loader.load_all()
self.runall(self.checks, sort=True)
stats = self.runner.stats
assert stats.num_cases(0) == 8
assert len(stats.failures()) == 2
for tf in stats.failures():
check = tf.testcase.check
exc_type, exc_value, _ = tf.exc_info
if check.name == 'T7':
assert isinstance(exc_value, TaskDependencyError)
# Check that cleanup is executed properly for successful tests as well
for t in stats.tasks():
check = t.testcase.check
if t.failed:
continue
if t.ref_count == 0:
assert os.path.exists(os.path.join(check.outputdir, 'out.txt'))
def test_dependencies_with_retries(self):
self.runner._max_retries = 2
self.test_dependencies()
def _make_loader(check_search_path):
return RegressionCheckLoader(check_search_path)
