class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources', 'unittests/foobar'], ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['badchecks'],
prefix=os.path.abspath('unittests/resources'))
self.system = System('foo')
self.resources = ResourcesManager()
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/emptycheck.py',
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/emptycheck.py'),
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources',
recurse=True,
system=self.system,
resources=self.resources)
self.assertEqual(11, len(checks))
def test_load_all(self):
checks = self.loader_with_path.load_all(system=self.system,
resources=self.resources)
self.assertEqual(10, len(checks))
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all(system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
def test_conflicted_checks(self):
self.loader_with_path._ignore_conflicts = False
self.assertRaises(NameConflictError,
self.loader_with_path.load_all,
system=self.system,
resources=self.resources)
def test_load_error(self):
self.assertRaises(OSError, self.loader.load_from_file,
'unittests/resources/foo.py')
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
warnings.simplefilter('ignore', ReframeDeprecationWarning)
self.loader = RegressionCheckLoader(['.'])
self.loader_with_path = RegressionCheckLoader(
['unittests/resources', 'unittests/foobar'])
self.loader_with_prefix = RegressionCheckLoader(
load_path=['badchecks'],
prefix=os.path.abspath('unittests/resources'))
self.system = System('foo')
self.resources = ResourcesManager()
def tearDown(self):
warnings.simplefilter('default', ReframeDeprecationWarning)
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/emptycheck.py',
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/emptycheck.py'),
system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'emptycheck')
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources',
recurse=True,
system=self.system,
resources=self.resources)
self.assertEqual(12, len(checks))
def test_load_all(self):
checks = self.loader_with_path.load_all(system=self.system,
resources=self.resources)
self.assertEqual(11, len(checks))
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all(system=self.system,
resources=self.resources)
self.assertEqual(1, len(checks))
def test_load_error(self):
self.assertRaises(ReframeError, self.loader.load_from_file,
'unittests/resources/foo.py')
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources/checks', 'unittests/foobar'],
ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['bad'],
prefix=os.path.abspath('unittests/resources/checks'))
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/checks/emptycheck.py')
assert 1 == len(checks)
assert checks[0].name == 'EmptyTest'
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/checks/emptycheck.py'))
assert 1 == len(checks)
assert checks[0].name == 'EmptyTest'
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources/checks',
recurse=True)
assert 12 == len(checks)
def test_load_all(self):
checks = self.loader_with_path.load_all()
assert 11 == len(checks)
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all()
assert 1 == len(checks)
def test_load_new_syntax(self):
checks = self.loader.load_from_file(
'unittests/resources/checks_unlisted/good.py')
assert 13 == len(checks)
def test_conflicted_checks(self):
self.loader_with_path._ignore_conflicts = False
with pytest.raises(NameConflictError):
self.loader_with_path.load_all()
def test_load_error(self):
with pytest.raises(OSError):
self.loader.load_from_file('unittests/resources/checks/foo.py')
def test_load_bad_required_version(self):
with pytest.raises(ValueError):
self.loader.load_from_file('unittests/resources/checks_unlisted/'
'no_required_version.py')
def test_load_bad_init(self):
tests = self.loader.load_from_file(
'unittests/resources/checks_unlisted/bad_init_check.py')
assert 0 == len(tests)
class TestRegressionTest(unittest.TestCase):
def setUp(self):
self.partition, self.prgenv = _setup_local_execution()
self.loader = RegressionCheckLoader(['unittests/resources/checks'])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
os_ext.rmtree('.rfm_testing', ignore_errors=True)
def replace_prefix(self, filename, new_prefix):
basename = os.path.basename(filename)
return os.path.join(new_prefix, basename)
def keep_files_list(self, test, compile_only=False):
ret = [
self.replace_prefix(sn.evaluate(test.stdout), test.outputdir),
self.replace_prefix(sn.evaluate(test.stderr), test.outputdir)
]
if not compile_only:
ret.append(
self.replace_prefix(test.job.script_filename, test.outputdir))
ret.extend(
[self.replace_prefix(f, test.outputdir) for f in test.keep_files])
return ret
def test_environ_setup(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
test.modules = ['testmod_foo']
test.variables = {'_FOO_': '1', '_BAR_': '2'}
test.local = True
test.setup(self.partition, self.prgenv)
for k in test.variables.keys():
assert k not in os.environ
def _run_test(self, test, compile_only=False):
_run(test, self.partition, self.prgenv)
assert not os.path.exists(test.stagedir)
for f in self.keep_files_list(test, compile_only):
assert os.path.exists(f)
@fixtures.switch_to_user_runtime
def test_hellocheck(self):
self.partition, self.prgenv = _setup_remote_execution()
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
self._run_test(test)
@fixtures.switch_to_user_runtime
def test_hellocheck_make(self):
self.partition, self.prgenv = _setup_remote_execution()
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck_make.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
self._run_test(test)
def test_hellocheck_local(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
# Test also the prebuild/postbuild functionality
test.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir']
test.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir']
test.keep_files = [
'prebuild', 'postbuild', 'prebuild_dir', 'postbuild_dir'
]
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_prepost_run(self):
@sn.sanity_function
def stagedir(test):
return test.stagedir
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.prgenv.name]
# Test also the prebuild/postbuild functionality
test.pre_run = ['echo prerun: `pwd`']
test.post_run = ['echo postrun: `pwd`']
pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1)
post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1)
test.sanity_patterns = sn.all([
sn.assert_eq(stagedir(test), pre_run_path),
sn.assert_eq(stagedir(test), post_run_path),
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_prepost_run_in_setup(self):
def custom_setup(obj, partition, environ, **job_opts):
super(obj.__class__, obj).setup(partition, environ, **job_opts)
obj.pre_run = ['echo Prerunning cmd from setup phase']
obj.post_run = ['echo Postruning cmd from setup phase']
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Monkey patch the setup method of the test
test.setup = custom_setup.__get__(test)
# Use test environment for the regression check
test.valid_prog_environs = ['*']
test.sanity_patterns = sn.all([
sn.assert_found(r'^Prerunning cmd from setup phase', test.stdout),
sn.assert_found(r'Hello, World\!', test.stdout),
sn.assert_found(r'^Postruning cmd from setup phase', test.stdout)
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_run_only_sanity(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.executable = './hello.sh'
self.executable_opts = ['Hello, World!']
self.local = True
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_run_only_no_srcdir(self):
@fixtures.custom_prefix('foo/bar/')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.executable = 'echo'
self.executable_opts = ['hello']
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'hello', self.stdout)
test = MyTest()
assert test.sourcesdir is None
self._run_test(MyTest())
def test_compile_only_failure(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.sourcepath = 'compiler_failure.c'
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
test = MyTest()
test.setup(self.partition, self.prgenv)
test.compile()
with pytest.raises(BuildError):
test.compile_wait()
def test_compile_only_warning(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.build_system = 'SingleSource'
self.build_system.srcfile = 'compiler_warning.c'
self.build_system.cflags = ['-Wall']
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'warning', self.stderr)
self._run_test(MyTest(), compile_only=True)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
def test_supports_system(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
test.valid_systems = ['*']
assert test.supports_system('gpu')
assert test.supports_system('login')
assert test.supports_system('testsys:gpu')
assert test.supports_system('testsys:login')
test.valid_systems = ['testsys']
assert test.supports_system('gpu')
assert test.supports_system('login')
assert test.supports_system('testsys:gpu')
assert test.supports_system('testsys:login')
test.valid_systems = ['testsys:gpu']
assert test.supports_system('gpu')
assert not test.supports_system('login')
assert test.supports_system('testsys:gpu')
assert not test.supports_system('testsys:login')
test.valid_systems = ['testsys:login']
assert not test.supports_system('gpu')
assert test.supports_system('login')
assert not test.supports_system('testsys:gpu')
assert test.supports_system('testsys:login')
test.valid_systems = ['foo']
assert not test.supports_system('gpu')
assert not test.supports_system('login')
assert not test.supports_system('testsys:gpu')
assert not test.supports_system('testsys:login')
def test_supports_environ(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
test.valid_prog_environs = ['*']
assert test.supports_environ('foo1')
assert test.supports_environ('foo-env')
assert test.supports_environ('*')
def test_sourcesdir_none(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RegressionTest):
def __init__(self):
self.sourcesdir = None
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
with pytest.raises(ReframeError):
self._run_test(MyTest())
def test_sourcesdir_build_system(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RegressionTest):
def __init__(self):
self.build_system = 'Make'
self.sourcepath = 'code'
self.executable = './code/hello'
self.local = True
self.valid_systems = ['*']
self.valid_prog_environs = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_sourcesdir_none_generated_sources(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RegressionTest):
def __init__(self):
self.sourcesdir = None
self.prebuild_cmd = [
"printf '#include <stdio.h>\\n int main(){ "
"printf(\"Hello, World!\\\\n\"); return 0; }' > hello.c"
]
self.executable = './hello'
self.sourcepath = 'hello.c'
self.local = True
self.valid_systems = ['*']
self.valid_prog_environs = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_sourcesdir_none_compile_only(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.sourcesdir = None
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
with pytest.raises(BuildError):
self._run_test(MyTest())
def test_sourcesdir_none_run_only(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.RunOnlyRegressionTest):
def __init__(self):
self.sourcesdir = None
self.executable = 'echo'
self.executable_opts = ["Hello, World!"]
self.local = True
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
self.sanity_patterns = sn.assert_found(r'Hello, World\!',
self.stdout)
self._run_test(MyTest())
def test_sourcepath_abs(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
test = MyTest()
test.setup(self.partition, self.prgenv)
test.sourcepath = '/usr/src'
with pytest.raises(PipelineError):
test.compile()
def test_sourcepath_upref(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(rfm.CompileOnlyRegressionTest):
def __init__(self):
self.valid_prog_environs = ['*']
self.valid_systems = ['*']
test = MyTest()
test.setup(self.partition, self.prgenv)
test.sourcepath = '../hellosrc'
with pytest.raises(PipelineError):
test.compile()
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
def test_extra_resources(self):
@fixtures.custom_prefix('unittests/resources/checks')
class MyTest(HelloTest):
def __init__(self):
super().__init__()
self.name = type(self).__name__
self.executable = os.path.join('.', self.name)
self.local = True
@rfm.run_after('setup')
def set_resources(self):
test.extra_resources = {
'gpu': {
'num_gpus_per_node': 2
},
'datawarp': {
'capacity': '100GB',
'stagein_src': test.stagedir
}
}
test.job.options += ['--foo']
test = MyTest()
partition = rt.runtime().system.partition('gpu')
environ = partition.environment('builtin-gcc')
_run(test, partition, environ)
expected_job_options = [
'--gres=gpu:2', '#DW jobdw capacity=100GB',
'#DW stage_in source=%s' % test.stagedir, '--foo'
]
self.assertCountEqual(expected_job_options, test.job.options)
class TestRegressionTest(unittest.TestCase):
def setup_local_execution(self):
self.partition = rt.runtime().system.partition('login')
self.progenv = self.partition.environment('builtin-gcc')
def setup_remote_execution(self):
self.partition = fixtures.partition_with_scheduler()
if self.partition is None:
self.skipTest('job submission not supported')
try:
self.progenv = self.partition.environs[0]
except IndexError:
self.skipTest('no environments configured for partition: %s' %
self.partition.fullname)
def setUp(self):
self.setup_local_execution()
self.loader = RegressionCheckLoader(['unittests/resources/checks'])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
os_ext.rmtree('.rfm_testing', ignore_errors=True)
def replace_prefix(self, filename, new_prefix):
basename = os.path.basename(filename)
return os.path.join(new_prefix, basename)
def keep_files_list(self, test, compile_only=False):
from reframe.core.deferrable import evaluate
ret = [
self.replace_prefix(evaluate(test.stdout), test.outputdir),
self.replace_prefix(evaluate(test.stderr), test.outputdir)
]
if not compile_only:
ret.append(
self.replace_prefix(test.job.script_filename, test.outputdir))
ret.extend(
[self.replace_prefix(f, test.outputdir) for f in test.keep_files])
return ret
def test_environ_setup(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
test.modules = ['testmod_foo']
test.variables = {'_FOO_': '1', '_BAR_': '2'}
test.local = True
test.setup(self.partition, self.progenv)
for k in test.variables.keys():
self.assertNotIn(k, os.environ)
# Manually unload the environment
self.progenv.unload()
def _run_test(self, test, compile_only=False):
test.setup(self.partition, self.progenv)
test.compile()
test.compile_wait()
test.run()
test.wait()
test.check_sanity()
test.check_performance()
test.cleanup(remove_files=True)
self.assertFalse(os.path.exists(test.stagedir))
for f in self.keep_files_list(test, compile_only):
self.assertTrue(os.path.exists(f))
@fixtures.switch_to_user_runtime
def test_hellocheck(self):
self.setup_remote_execution()
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
self._run_test(test)
@fixtures.switch_to_user_runtime
def test_hellocheck_make(self):
self.setup_remote_execution()
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck_make.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
self._run_test(test)
def test_hellocheck_local(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
# Test also the prebuild/postbuild functionality
test.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir']
test.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir']
test.keep_files = [
'prebuild', 'postbuild', 'prebuild_dir', 'postbuild_dir'
]
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_prepost_run(self):
@sn.sanity_function
def stagedir(test):
return test.stagedir
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
# Test also the prebuild/postbuild functionality
test.pre_run = ['echo prerun: `pwd`']
test.post_run = ['echo postrun: `pwd`']
pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1)
post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1)
test.sanity_patterns = sn.all([
sn.assert_eq(stagedir(test), pre_run_path),
sn.assert_eq(stagedir(test), post_run_path),
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_prepost_run_in_setup(self):
def custom_setup(obj, partition, environ, **job_opts):
super(obj.__class__, obj).setup(partition, environ, **job_opts)
obj.pre_run = ['echo Prerunning cmd from setup phase']
obj.post_run = ['echo Postruning cmd from setup phase']
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
# Monkey patch the setup method of the test
test.setup = custom_setup.__get__(test)
# Use test environment for the regression check
test.valid_prog_environs = ['*']
test.sanity_patterns = sn.all([
sn.assert_found(r'^Prerunning cmd from setup phase', test.stdout),
sn.assert_found(r'Hello, World\!', test.stdout),
sn.assert_found(r'^Postruning cmd from setup phase', test.stdout)
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_run_only_sanity(self):
test = RunOnlyRegressionTest('runonlycheck',
'unittests/resources/checks')
test.executable = './hello.sh'
test.executable_opts = ['Hello, World!']
test.local = True
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout)
self._run_test(test)
def test_compile_only_failure(self):
test = CompileOnlyRegressionTest('compileonlycheck',
'unittests/resources/checks')
test.sourcepath = 'compiler_failure.c'
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.setup(self.partition, self.progenv)
test.compile()
self.assertRaises(BuildError, test.compile_wait)
def test_compile_only_warning(self):
test = CompileOnlyRegressionTest('compileonlycheckwarning',
'unittests/resources/checks')
test.build_system = 'SingleSource'
test.build_system.srcfile = 'compiler_warning.c'
test.build_system.cflags = ['-Wall']
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.sanity_patterns = sn.assert_found(r'warning', test.stderr)
self._run_test(test, compile_only=True)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
def test_supports_system(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
test.valid_systems = ['*']
self.assertTrue(test.supports_system('gpu'))
self.assertTrue(test.supports_system('login'))
self.assertTrue(test.supports_system('testsys:gpu'))
self.assertTrue(test.supports_system('testsys:login'))
test.valid_systems = ['testsys']
self.assertTrue(test.supports_system('gpu'))
self.assertTrue(test.supports_system('login'))
self.assertTrue(test.supports_system('testsys:gpu'))
self.assertTrue(test.supports_system('testsys:login'))
test.valid_systems = ['testsys:gpu']
self.assertTrue(test.supports_system('gpu'))
self.assertFalse(test.supports_system('login'))
self.assertTrue(test.supports_system('testsys:gpu'))
self.assertFalse(test.supports_system('testsys:login'))
test.valid_systems = ['testsys:login']
self.assertFalse(test.supports_system('gpu'))
self.assertTrue(test.supports_system('login'))
self.assertFalse(test.supports_system('testsys:gpu'))
self.assertTrue(test.supports_system('testsys:login'))
test.valid_systems = ['foo']
self.assertFalse(test.supports_system('gpu'))
self.assertFalse(test.supports_system('login'))
self.assertFalse(test.supports_system('testsys:gpu'))
self.assertFalse(test.supports_system('testsys:login'))
def test_supports_environ(self):
test = self.loader.load_from_file(
'unittests/resources/checks/hellocheck.py')[0]
test.valid_prog_environs = ['*']
self.assertTrue(test.supports_environ('foo1'))
self.assertTrue(test.supports_environ('foo-env'))
self.assertTrue(test.supports_environ('*'))
test.valid_prog_environs = ['PrgEnv-foo-*']
self.assertTrue(test.supports_environ('PrgEnv-foo-version1'))
self.assertTrue(test.supports_environ('PrgEnv-foo-version2'))
self.assertFalse(test.supports_environ('PrgEnv-boo-version1'))
self.assertFalse(test.supports_environ('Prgenv-foo-version1'))
def test_sourcesdir_none(self):
test = RegressionTest('hellocheck', 'unittests/resources/checks')
test.sourcesdir = None
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
self.assertRaises(ReframeError, self._run_test, test)
def test_sourcesdir_none_generated_sources(self):
test = RegressionTest('hellocheck_generated_sources',
'unittests/resources/checks')
test.sourcesdir = None
test.prebuild_cmd = [
"printf '#include <stdio.h>\\n int main(){ "
"printf(\"Hello, World!\\\\n\"); return 0; }' "
"> hello.c"
]
test.executable = './hello'
test.sourcepath = 'hello.c'
test.local = True
test.valid_systems = ['*']
test.valid_prog_environs = ['*']
test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout)
self._run_test(test)
def test_sourcesdir_none_compile_only(self):
test = CompileOnlyRegressionTest('hellocheck',
'unittests/resources/checks')
test.sourcesdir = None
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
self.assertRaises(BuildError, self._run_test, test)
def test_sourcesdir_none_run_only(self):
test = RunOnlyRegressionTest('hellocheck',
'unittests/resources/checks')
test.sourcesdir = None
test.executable = 'echo'
test.executable_opts = ["Hello, World!"]
test.local = True
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout)
self._run_test(test)
def test_sourcepath_abs(self):
test = CompileOnlyRegressionTest('compileonlycheck',
'unittests/resources/checks')
test.valid_prog_environs = [self.progenv.name]
test.valid_systems = ['*']
test.setup(self.partition, self.progenv)
test.sourcepath = '/usr/src'
self.assertRaises(PipelineError, test.compile)
def test_sourcepath_upref(self):
test = CompileOnlyRegressionTest('compileonlycheck',
'unittests/resources/checks')
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.setup(self.partition, self.progenv)
test.sourcepath = '../hellosrc'
self.assertRaises(PipelineError, test.compile)
@rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
def test_extra_resources(self):
# Load test site configuration
test = RegressionTest('dummycheck', 'unittests/resources/checks')
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.extra_resources = {
'gpu': {
'num_gpus_per_node': 2
},
'datawarp': {
'capacity': '100GB',
'stagein_src': '/foo'
}
}
partition = rt.runtime().system.partition('gpu')
environ = partition.environment('builtin-gcc')
test.setup(partition, environ)
test.job.options += ['--foo']
expected_job_options = [
'--gres=gpu:2', '#DW jobdw capacity=100GB',
'#DW stage_in source=/foo', '--foo'
]
self.assertCountEqual(expected_job_options, test.job.options)
class TestRegressionTest(unittest.TestCase):
def setUp(self):
get_modules_system().searchpath_add(fixtures.TEST_MODULES)
# Load a system configuration
self.system, self.partition, self.progenv = fixtures.get_test_config()
self.resourcesdir = tempfile.mkdtemp(dir='unittests')
self.loader = RegressionCheckLoader(['unittests/resources'])
self.resources = ResourcesManager(prefix=self.resourcesdir)
def tearDown(self):
shutil.rmtree(self.resourcesdir, ignore_errors=True)
def setup_from_site(self):
self.partition = fixtures.partition_with_scheduler(None)
# pick the first environment of partition
if self.partition.environs:
self.progenv = self.partition.environs[0]
def replace_prefix(self, filename, new_prefix):
basename = os.path.basename(filename)
return os.path.join(new_prefix, basename)
def keep_files_list(self, test, compile_only=False):
from reframe.core.deferrable import evaluate
ret = [
self.replace_prefix(evaluate(test.stdout), test.outputdir),
self.replace_prefix(evaluate(test.stderr), test.outputdir)
]
if not compile_only:
ret.append(
self.replace_prefix(test.job.script_filename, test.outputdir))
ret.extend(
[self.replace_prefix(f, test.outputdir) for f in test.keep_files])
return ret
def test_environ_setup(self):
test = self.loader.load_from_file('unittests/resources/hellocheck.py',
system=self.system,
resources=self.resources)[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
test.modules = ['testmod_foo']
test.variables = {'_FOO_': '1', '_BAR_': '2'}
test.local = True
test.setup(self.partition, self.progenv)
for m in test.modules:
self.assertTrue(get_modules_system().is_module_loaded(m))
for k, v in test.variables.items():
self.assertEqual(os.environ[k], v)
# Manually unload the environment
self.progenv.unload()
def _run_test(self, test, compile_only=False):
test.setup(self.partition, self.progenv)
test.compile()
test.run()
test.wait()
test.check_sanity()
test.check_performance()
test.cleanup(remove_files=True)
self.assertFalse(os.path.exists(test.stagedir))
for f in self.keep_files_list(test, compile_only):
self.assertTrue(os.path.exists(f))
@unittest.skipIf(not fixtures.partition_with_scheduler(None),
'job submission not supported')
def test_hellocheck(self):
self.setup_from_site()
test = self.loader.load_from_file('unittests/resources/hellocheck.py',
system=self.system,
resources=self.resources)[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
self._run_test(test)
@unittest.skipIf(not fixtures.partition_with_scheduler(None),
'job submission not supported')
def test_hellocheck_make(self):
self.setup_from_site()
test = self.loader.load_from_file(
'unittests/resources/hellocheck_make.py',
system=self.system,
resources=self.resources)[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
self._run_test(test)
def test_hellocheck_local(self):
test = self.loader.load_from_file('unittests/resources/hellocheck.py',
system=self.system,
resources=self.resources)[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
# Test also the prebuild/postbuild functionality
test.prebuild_cmd = ['touch prebuild']
test.postbuild_cmd = ['touch postbuild']
test.keepfiles = ['prebuild', 'postbuild']
# Force local execution of the test
test.local = True
self._run_test(test)
def test_hellocheck_local_slashes(self):
# Try to fool path creation by adding slashes to environment partitions
# names
from reframe.core.environments import ProgEnvironment
self.progenv = ProgEnvironment('bad/name', self.progenv.modules,
self.progenv.variables)
# That's a bit hacky, but we are in a unit test
self.system._name += os.sep + 'bad'
self.partition._name += os.sep + 'bad'
self.test_hellocheck_local()
def test_hellocheck_local_prepost_run(self):
@sn.sanity_function
def stagedir(test):
return test.stagedir
test = self.loader.load_from_file('unittests/resources/hellocheck.py',
system=self.system,
resources=self.resources)[0]
# Use test environment for the regression check
test.valid_prog_environs = [self.progenv.name]
# Test also the prebuild/postbuild functionality
test.pre_run = ['echo prerun: `pwd`']
test.post_run = ['echo postrun: `pwd`']
pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1)
post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1)
test.sanity_patterns = sn.all([
sn.assert_eq(stagedir(test), pre_run_path),
sn.assert_eq(stagedir(test), post_run_path),
])
# Force local execution of the test
test.local = True
self._run_test(test)
def test_run_only_sanity(self):
test = RunOnlyRegressionTest('runonlycheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.executable = './hello.sh'
test.executable_opts = ['Hello, World!']
test.local = True
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout)
self._run_test(test)
def test_compile_only_failure(self):
test = CompileOnlyRegressionTest('compileonlycheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.sourcepath = 'compiler_failure.c'
test.valid_prog_environs = [self.progenv.name]
test.valid_systems = [self.system.name]
test.setup(self.partition, self.progenv)
self.assertRaises(CompilationError, test.compile)
def test_compile_only_warning(self):
test = CompileOnlyRegressionTest('compileonlycheckwarning',
'unittests/resources',
resources=self.resources,
system=self.system)
test.sourcepath = 'compiler_warning.c'
self.progenv.cflags = '-Wall'
test.valid_prog_environs = [self.progenv.name]
test.valid_systems = [self.system.name]
test.sanity_patterns = sn.assert_found(r'warning', test.stderr)
self._run_test(test, compile_only=True)
def test_supports_system(self):
test = self.loader.load_from_file('unittests/resources/hellocheck.py',
system=self.system,
resources=self.resources)[0]
test.valid_systems = ['*']
self.assertTrue(test.supports_system('gpu'))
self.assertTrue(test.supports_system('login'))
self.assertTrue(test.supports_system('testsys:gpu'))
self.assertTrue(test.supports_system('testsys:login'))
test.valid_systems = ['testsys']
self.assertTrue(test.supports_system('gpu'))
self.assertTrue(test.supports_system('login'))
self.assertTrue(test.supports_system('testsys:gpu'))
self.assertTrue(test.supports_system('testsys:login'))
test.valid_systems = ['testsys:gpu']
self.assertTrue(test.supports_system('gpu'))
self.assertFalse(test.supports_system('login'))
self.assertTrue(test.supports_system('testsys:gpu'))
self.assertFalse(test.supports_system('testsys:login'))
test.valid_systems = ['testsys:login']
self.assertFalse(test.supports_system('gpu'))
self.assertTrue(test.supports_system('login'))
self.assertFalse(test.supports_system('testsys:gpu'))
self.assertTrue(test.supports_system('testsys:login'))
test.valid_systems = ['foo']
self.assertFalse(test.supports_system('gpu'))
self.assertFalse(test.supports_system('login'))
self.assertFalse(test.supports_system('testsys:gpu'))
self.assertFalse(test.supports_system('testsys:login'))
def test_supports_environ(self):
test = self.loader.load_from_file('unittests/resources/hellocheck.py',
system=self.system,
resources=self.resources)[0]
test.valid_prog_environs = ['*']
self.assertTrue(test.supports_environ('foo1'))
self.assertTrue(test.supports_environ('foo-env'))
self.assertTrue(test.supports_environ('*'))
test.valid_prog_environs = ['PrgEnv-foo-*']
self.assertTrue(test.supports_environ('PrgEnv-foo-version1'))
self.assertTrue(test.supports_environ('PrgEnv-foo-version2'))
self.assertFalse(test.supports_environ('PrgEnv-boo-version1'))
self.assertFalse(test.supports_environ('Prgenv-foo-version1'))
def test_sourcesdir_none(self):
test = RegressionTest('hellocheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.sourcesdir = None
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
self.assertRaises(ReframeError, self._run_test, test)
def test_sourcesdir_none_generated_sources(self):
test = RegressionTest('hellocheck_generated_sources',
'unittests/resources',
resources=self.resources,
system=self.system)
test.sourcesdir = None
test.prebuild_cmd = [
"printf '#include <stdio.h>\\n int main(){ "
"printf(\"Hello, World!\\\\n\"); return 0; }' "
"> hello.c"
]
test.executable = './hello'
test.sourcepath = 'hello.c'
test.local = True
test.valid_systems = ['*']
test.valid_prog_environs = ['*']
test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout)
self._run_test(test)
def test_sourcesdir_none_compile_only(self):
test = CompileOnlyRegressionTest('hellocheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.sourcesdir = None
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
self.assertRaises(CompilationError, self._run_test, test)
def test_sourcesdir_none_run_only(self):
test = RunOnlyRegressionTest('hellocheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.sourcesdir = None
test.executable = 'echo'
test.executable_opts = ["Hello, World!"]
test.local = True
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout)
self._run_test(test)
def test_sourcepath_abs(self):
test = CompileOnlyRegressionTest('compileonlycheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.valid_prog_environs = [self.progenv.name]
test.valid_systems = [self.system.name]
test.setup(self.partition, self.progenv)
test.sourcepath = '/usr/src'
self.assertRaises(PipelineError, test.compile)
def test_sourcepath_upref(self):
test = CompileOnlyRegressionTest('compileonlycheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.valid_prog_environs = [self.progenv.name]
test.valid_systems = [self.system.name]
test.setup(self.partition, self.progenv)
test.sourcepath = '../hellosrc'
self.assertRaises(PipelineError, test.compile)
def test_extra_resources(self):
# Load test site configuration
system, partition, progenv = fixtures.get_test_config()
test = RegressionTest('dummycheck',
'unittests/resources',
resources=self.resources,
system=self.system)
test.valid_prog_environs = ['*']
test.valid_systems = ['*']
test.extra_resources = {
'gpu': {
'num_gpus_per_node': 2
},
'datawarp': {
'capacity': '100GB',
'stagein_src': '/foo'
}
}
test.setup(self.partition, self.progenv)
test.job.options += ['--foo']
expected_job_options = [
'--gres=gpu:2', '#DW jobdw capacity=100GB',
'#DW stage_in source=/foo', '--foo'
]
self.assertCountEqual(expected_job_options, test.job.options)
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources/checks', 'unittests/foobar'],
ignore_conflicts=True)
self.loader_with_prefix = RegressionCheckLoader(
load_path=['bad'],
prefix=os.path.abspath('unittests/resources/checks'))
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/checks/emptycheck.py')
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'EmptyTest')
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/checks/emptycheck.py'))
self.assertEqual(1, len(checks))
self.assertEqual(checks[0].name, 'EmptyTest')
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources/checks',
recurse=True)
self.assertEqual(11, len(checks))
def test_load_all(self):
checks = self.loader_with_path.load_all()
self.assertEqual(10, len(checks))
def test_load_all_with_prefix(self):
checks = self.loader_with_prefix.load_all()
self.assertEqual(1, len(checks))
def test_load_new_syntax(self):
checks = self.loader.load_from_file(
'unittests/resources/checks_unlisted/good.py')
self.assertEqual(13, len(checks))
def test_load_mixed_syntax(self):
self.assertRaises(RegressionTestLoadError, self.loader.load_from_file,
'unittests/resources/checks_unlisted/mixed.py')
def test_conflicted_checks(self):
self.loader_with_path._ignore_conflicts = False
self.assertRaises(NameConflictError, self.loader_with_path.load_all)
def test_load_error(self):
self.assertRaises(OSError, self.loader.load_from_file,
'unittests/resources/checks/foo.py')
def test_load_bad_required_version(self):
with self.assertRaises(ValueError):
self.loader.load_from_file('unittests/resources/checks_unlisted/'
'no_required_version.py')
def test_load_bad_init(self):
tests = self.loader.load_from_file(
'unittests/resources/checks_unlisted/bad_init_check.py')
self.assertEqual(0, len(tests))
def load_test(testfile):
loader = RegressionCheckLoader(['unittests/resources/checks'])
return loader.load_from_file(testfile)
class TestRegressionCheckLoader(unittest.TestCase):
def setUp(self):
self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True)
self.loader_with_path = RegressionCheckLoader(
['unittests/resources/checks', 'unittests/foobar'],
ignore_conflicts=True)
def test_load_file_relative(self):
checks = self.loader.load_from_file(
'unittests/resources/checks/emptycheck.py')
assert 1 == len(checks)
assert checks[0].name == 'EmptyTest'
def test_load_file_absolute(self):
checks = self.loader.load_from_file(
os.path.abspath('unittests/resources/checks/emptycheck.py'))
assert 1 == len(checks)
assert checks[0].name == 'EmptyTest'
def test_load_recursive(self):
checks = self.loader.load_from_dir('unittests/resources/checks',
recurse=True)
assert 12 == len(checks)
def test_load_all(self):
checks = self.loader_with_path.load_all()
assert 11 == len(checks)
def test_load_new_syntax(self):
checks = self.loader.load_from_file(
'unittests/resources/checks_unlisted/good.py')
assert 13 == len(checks)
def test_conflicted_checks(self):
self.loader_with_path._ignore_conflicts = False
with pytest.raises(NameConflictError):
self.loader_with_path.load_all()
def test_load_error(self):
with pytest.raises(OSError):
self.loader.load_from_file('unittests/resources/checks/foo.py')
def test_load_bad_required_version(self):
with pytest.raises(ValueError):
self.loader.load_from_file('unittests/resources/checks_unlisted/'
'no_required_version.py')
def test_load_bad_init(self):
tests = self.loader.load_from_file(
'unittests/resources/checks_unlisted/bad_init_check.py')
assert 0 == len(tests)
def test_special_test(self):
with pytest.warns(ReframeDeprecationWarning):
@rfm.simple_test
class TestDeprecated(rfm.RegressionTest):
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
with pytest.warns(ReframeDeprecationWarning):
@rfm.simple_test
class TestDeprecatedRunOnly(rfm.RunOnlyRegressionTest):
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
with pytest.warns(ReframeDeprecationWarning):
@rfm.simple_test
class TestDeprecatedCompileOnly(rfm.CompileOnlyRegressionTest):
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
with pytest.warns(ReframeDeprecationWarning):
@rfm.simple_test
class TestDeprecatedCompileOnlyDerived(TestDeprecatedCompileOnly):
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
with pytest.warns(None) as warnings:
@rfm.simple_test
class TestSimple(rfm.RegressionTest):
def __init__(self):
pass
@rfm.simple_test
class TestSpecial(rfm.RegressionTest, special=True):
def __init__(self):
pass
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
@rfm.simple_test
class TestSpecialRunOnly(rfm.RunOnlyRegressionTest, special=True):
def __init__(self):
pass
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
def run(self):
super().run()
@rfm.simple_test
class TestSpecialCompileOnly(rfm.CompileOnlyRegressionTest,
special=True):
def __init__(self):
pass
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
def run(self):
super().run()
assert not any(
isinstance(w.message, ReframeDeprecationWarning) for w in warnings)
with pytest.warns(ReframeDeprecationWarning) as warnings:
@rfm.simple_test
class TestSpecialDerived(TestSpecial):
def __init__(self):
pass
def setup(self, partition, environ, **job_opts):
super().setup(system, environ, **job_opts)
def run(self):
super().run()
assert len(warnings) == 2
@rfm.simple_test
class TestFinal(rfm.RegressionTest):
def __init__(self):
pass
@rfm.final
def my_new_final(seld):
pass
with pytest.warns(ReframeDeprecationWarning):
@rfm.simple_test
class TestFinalDerived(TestFinal):
def my_new_final(self, a, b):
pass
