class TestRegressionCheckLoader(unittest.TestCase): def setUp(self): self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True) self.loader_with_path = RegressionCheckLoader( ['unittests/resources', 'unittests/foobar'], ignore_conflicts=True) self.loader_with_prefix = RegressionCheckLoader( load_path=['badchecks'], prefix=os.path.abspath('unittests/resources')) self.system = System('foo') self.resources = ResourcesManager() def test_load_file_relative(self): checks = self.loader.load_from_file( 'unittests/resources/emptycheck.py', system=self.system, resources=self.resources) self.assertEqual(1, len(checks)) self.assertEqual(checks[0].name, 'emptycheck') def test_load_file_absolute(self): checks = self.loader.load_from_file( os.path.abspath('unittests/resources/emptycheck.py'), system=self.system, resources=self.resources) self.assertEqual(1, len(checks)) self.assertEqual(checks[0].name, 'emptycheck') def test_load_recursive(self): checks = self.loader.load_from_dir('unittests/resources', recurse=True, system=self.system, resources=self.resources) self.assertEqual(11, len(checks)) def test_load_all(self): checks = self.loader_with_path.load_all(system=self.system, resources=self.resources) self.assertEqual(10, len(checks)) def test_load_all_with_prefix(self): checks = self.loader_with_prefix.load_all(system=self.system, resources=self.resources) self.assertEqual(1, len(checks)) def test_conflicted_checks(self): self.loader_with_path._ignore_conflicts = False self.assertRaises(NameConflictError, self.loader_with_path.load_all, system=self.system, resources=self.resources) def test_load_error(self): self.assertRaises(OSError, self.loader.load_from_file, 'unittests/resources/foo.py')
class TestRegressionCheckLoader(unittest.TestCase): def setUp(self): warnings.simplefilter('ignore', ReframeDeprecationWarning) self.loader = RegressionCheckLoader(['.']) self.loader_with_path = RegressionCheckLoader( ['unittests/resources', 'unittests/foobar']) self.loader_with_prefix = RegressionCheckLoader( load_path=['badchecks'], prefix=os.path.abspath('unittests/resources')) self.system = System('foo') self.resources = ResourcesManager() def tearDown(self): warnings.simplefilter('default', ReframeDeprecationWarning) def test_load_file_relative(self): checks = self.loader.load_from_file( 'unittests/resources/emptycheck.py', system=self.system, resources=self.resources) self.assertEqual(1, len(checks)) self.assertEqual(checks[0].name, 'emptycheck') def test_load_file_absolute(self): checks = self.loader.load_from_file( os.path.abspath('unittests/resources/emptycheck.py'), system=self.system, resources=self.resources) self.assertEqual(1, len(checks)) self.assertEqual(checks[0].name, 'emptycheck') def test_load_recursive(self): checks = self.loader.load_from_dir('unittests/resources', recurse=True, system=self.system, resources=self.resources) self.assertEqual(12, len(checks)) def test_load_all(self): checks = self.loader_with_path.load_all(system=self.system, resources=self.resources) self.assertEqual(11, len(checks)) def test_load_all_with_prefix(self): checks = self.loader_with_prefix.load_all(system=self.system, resources=self.resources) self.assertEqual(1, len(checks)) def test_load_error(self): self.assertRaises(ReframeError, self.loader.load_from_file, 'unittests/resources/foo.py')
class TestRegressionCheckLoader(unittest.TestCase): def setUp(self): self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True) self.loader_with_path = RegressionCheckLoader( ['unittests/resources/checks', 'unittests/foobar'], ignore_conflicts=True) self.loader_with_prefix = RegressionCheckLoader( load_path=['bad'], prefix=os.path.abspath('unittests/resources/checks')) def test_load_file_relative(self): checks = self.loader.load_from_file( 'unittests/resources/checks/emptycheck.py') assert 1 == len(checks) assert checks[0].name == 'EmptyTest' def test_load_file_absolute(self): checks = self.loader.load_from_file( os.path.abspath('unittests/resources/checks/emptycheck.py')) assert 1 == len(checks) assert checks[0].name == 'EmptyTest' def test_load_recursive(self): checks = self.loader.load_from_dir('unittests/resources/checks', recurse=True) assert 12 == len(checks) def test_load_all(self): checks = self.loader_with_path.load_all() assert 11 == len(checks) def test_load_all_with_prefix(self): checks = self.loader_with_prefix.load_all() assert 1 == len(checks) def test_load_new_syntax(self): checks = self.loader.load_from_file( 'unittests/resources/checks_unlisted/good.py') assert 13 == len(checks) def test_conflicted_checks(self): self.loader_with_path._ignore_conflicts = False with pytest.raises(NameConflictError): self.loader_with_path.load_all() def test_load_error(self): with pytest.raises(OSError): self.loader.load_from_file('unittests/resources/checks/foo.py') def test_load_bad_required_version(self): with pytest.raises(ValueError): self.loader.load_from_file('unittests/resources/checks_unlisted/' 'no_required_version.py') def test_load_bad_init(self): tests = self.loader.load_from_file( 'unittests/resources/checks_unlisted/bad_init_check.py') assert 0 == len(tests)
class TestRegressionTest(unittest.TestCase): def setUp(self): self.partition, self.prgenv = _setup_local_execution() self.loader = RegressionCheckLoader(['unittests/resources/checks']) # Set runtime prefix rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') def tearDown(self): os_ext.rmtree(rt.runtime().resources.prefix) os_ext.rmtree('.rfm_testing', ignore_errors=True) def replace_prefix(self, filename, new_prefix): basename = os.path.basename(filename) return os.path.join(new_prefix, basename) def keep_files_list(self, test, compile_only=False): ret = [ self.replace_prefix(sn.evaluate(test.stdout), test.outputdir), self.replace_prefix(sn.evaluate(test.stderr), test.outputdir) ] if not compile_only: ret.append( self.replace_prefix(test.job.script_filename, test.outputdir)) ret.extend( [self.replace_prefix(f, test.outputdir) for f in test.keep_files]) return ret def test_environ_setup(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.prgenv.name] test.modules = ['testmod_foo'] test.variables = {'_FOO_': '1', '_BAR_': '2'} test.local = True test.setup(self.partition, self.prgenv) for k in test.variables.keys(): assert k not in os.environ def _run_test(self, test, compile_only=False): _run(test, self.partition, self.prgenv) assert not os.path.exists(test.stagedir) for f in self.keep_files_list(test, compile_only): assert os.path.exists(f) @fixtures.switch_to_user_runtime def test_hellocheck(self): self.partition, self.prgenv = _setup_remote_execution() test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.prgenv.name] self._run_test(test) @fixtures.switch_to_user_runtime def test_hellocheck_make(self): self.partition, self.prgenv = _setup_remote_execution() test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck_make.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.prgenv.name] self._run_test(test) def test_hellocheck_local(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.prgenv.name] # Test also the prebuild/postbuild functionality test.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir'] test.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir'] test.keep_files = [ 'prebuild', 'postbuild', 'prebuild_dir', 'postbuild_dir' ] # Force local execution of the test test.local = True self._run_test(test) def test_hellocheck_local_prepost_run(self): @sn.sanity_function def stagedir(test): return test.stagedir test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.prgenv.name] # Test also the prebuild/postbuild functionality test.pre_run = ['echo prerun: `pwd`'] test.post_run = ['echo postrun: `pwd`'] pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1) post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1) test.sanity_patterns = sn.all([ sn.assert_eq(stagedir(test), pre_run_path), sn.assert_eq(stagedir(test), post_run_path), ]) # Force local execution of the test test.local = True self._run_test(test) def test_hellocheck_local_prepost_run_in_setup(self): def custom_setup(obj, partition, environ, **job_opts): super(obj.__class__, obj).setup(partition, environ, **job_opts) obj.pre_run = ['echo Prerunning cmd from setup phase'] obj.post_run = ['echo Postruning cmd from setup phase'] test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Monkey patch the setup method of the test test.setup = custom_setup.__get__(test) # Use test environment for the regression check test.valid_prog_environs = ['*'] test.sanity_patterns = sn.all([ sn.assert_found(r'^Prerunning cmd from setup phase', test.stdout), sn.assert_found(r'Hello, World\!', test.stdout), sn.assert_found(r'^Postruning cmd from setup phase', test.stdout) ]) # Force local execution of the test test.local = True self._run_test(test) def test_run_only_sanity(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.RunOnlyRegressionTest): def __init__(self): self.executable = './hello.sh' self.executable_opts = ['Hello, World!'] self.local = True self.valid_prog_environs = ['*'] self.valid_systems = ['*'] self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) self._run_test(MyTest()) def test_run_only_no_srcdir(self): @fixtures.custom_prefix('foo/bar/') class MyTest(rfm.RunOnlyRegressionTest): def __init__(self): self.executable = 'echo' self.executable_opts = ['hello'] self.valid_prog_environs = ['*'] self.valid_systems = ['*'] self.sanity_patterns = sn.assert_found(r'hello', self.stdout) test = MyTest() assert test.sourcesdir is None self._run_test(MyTest()) def test_compile_only_failure(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.CompileOnlyRegressionTest): def __init__(self): self.sourcepath = 'compiler_failure.c' self.valid_prog_environs = ['*'] self.valid_systems = ['*'] test = MyTest() test.setup(self.partition, self.prgenv) test.compile() with pytest.raises(BuildError): test.compile_wait() def test_compile_only_warning(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.RunOnlyRegressionTest): def __init__(self): self.build_system = 'SingleSource' self.build_system.srcfile = 'compiler_warning.c' self.build_system.cflags = ['-Wall'] self.valid_prog_environs = ['*'] self.valid_systems = ['*'] self.sanity_patterns = sn.assert_found(r'warning', self.stderr) self._run_test(MyTest(), compile_only=True) @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def test_supports_system(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] test.valid_systems = ['*'] assert test.supports_system('gpu') assert test.supports_system('login') assert test.supports_system('testsys:gpu') assert test.supports_system('testsys:login') test.valid_systems = ['testsys'] assert test.supports_system('gpu') assert test.supports_system('login') assert test.supports_system('testsys:gpu') assert test.supports_system('testsys:login') test.valid_systems = ['testsys:gpu'] assert test.supports_system('gpu') assert not test.supports_system('login') assert test.supports_system('testsys:gpu') assert not test.supports_system('testsys:login') test.valid_systems = ['testsys:login'] assert not test.supports_system('gpu') assert test.supports_system('login') assert not test.supports_system('testsys:gpu') assert test.supports_system('testsys:login') test.valid_systems = ['foo'] assert not test.supports_system('gpu') assert not test.supports_system('login') assert not test.supports_system('testsys:gpu') assert not test.supports_system('testsys:login') def test_supports_environ(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] test.valid_prog_environs = ['*'] assert test.supports_environ('foo1') assert test.supports_environ('foo-env') assert test.supports_environ('*') def test_sourcesdir_none(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.RegressionTest): def __init__(self): self.sourcesdir = None self.valid_prog_environs = ['*'] self.valid_systems = ['*'] with pytest.raises(ReframeError): self._run_test(MyTest()) def test_sourcesdir_build_system(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.RegressionTest): def __init__(self): self.build_system = 'Make' self.sourcepath = 'code' self.executable = './code/hello' self.local = True self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) self._run_test(MyTest()) def test_sourcesdir_none_generated_sources(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.RegressionTest): def __init__(self): self.sourcesdir = None self.prebuild_cmd = [ "printf '#include <stdio.h>\\n int main(){ " "printf(\"Hello, World!\\\\n\"); return 0; }' > hello.c" ] self.executable = './hello' self.sourcepath = 'hello.c' self.local = True self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) self._run_test(MyTest()) def test_sourcesdir_none_compile_only(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.CompileOnlyRegressionTest): def __init__(self): self.sourcesdir = None self.valid_prog_environs = ['*'] self.valid_systems = ['*'] with pytest.raises(BuildError): self._run_test(MyTest()) def test_sourcesdir_none_run_only(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.RunOnlyRegressionTest): def __init__(self): self.sourcesdir = None self.executable = 'echo' self.executable_opts = ["Hello, World!"] self.local = True self.valid_prog_environs = ['*'] self.valid_systems = ['*'] self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) self._run_test(MyTest()) def test_sourcepath_abs(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.CompileOnlyRegressionTest): def __init__(self): self.valid_prog_environs = ['*'] self.valid_systems = ['*'] test = MyTest() test.setup(self.partition, self.prgenv) test.sourcepath = '/usr/src' with pytest.raises(PipelineError): test.compile() def test_sourcepath_upref(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(rfm.CompileOnlyRegressionTest): def __init__(self): self.valid_prog_environs = ['*'] self.valid_systems = ['*'] test = MyTest() test.setup(self.partition, self.prgenv) test.sourcepath = '../hellosrc' with pytest.raises(PipelineError): test.compile() @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def test_extra_resources(self): @fixtures.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): def __init__(self): super().__init__() self.name = type(self).__name__ self.executable = os.path.join('.', self.name) self.local = True @rfm.run_after('setup') def set_resources(self): test.extra_resources = { 'gpu': { 'num_gpus_per_node': 2 }, 'datawarp': { 'capacity': '100GB', 'stagein_src': test.stagedir } } test.job.options += ['--foo'] test = MyTest() partition = rt.runtime().system.partition('gpu') environ = partition.environment('builtin-gcc') _run(test, partition, environ) expected_job_options = [ '--gres=gpu:2', '#DW jobdw capacity=100GB', '#DW stage_in source=%s' % test.stagedir, '--foo' ] self.assertCountEqual(expected_job_options, test.job.options)
class TestRegressionTest(unittest.TestCase): def setup_local_execution(self): self.partition = rt.runtime().system.partition('login') self.progenv = self.partition.environment('builtin-gcc') def setup_remote_execution(self): self.partition = fixtures.partition_with_scheduler() if self.partition is None: self.skipTest('job submission not supported') try: self.progenv = self.partition.environs[0] except IndexError: self.skipTest('no environments configured for partition: %s' % self.partition.fullname) def setUp(self): self.setup_local_execution() self.loader = RegressionCheckLoader(['unittests/resources/checks']) # Set runtime prefix rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') def tearDown(self): os_ext.rmtree(rt.runtime().resources.prefix) os_ext.rmtree('.rfm_testing', ignore_errors=True) def replace_prefix(self, filename, new_prefix): basename = os.path.basename(filename) return os.path.join(new_prefix, basename) def keep_files_list(self, test, compile_only=False): from reframe.core.deferrable import evaluate ret = [ self.replace_prefix(evaluate(test.stdout), test.outputdir), self.replace_prefix(evaluate(test.stderr), test.outputdir) ] if not compile_only: ret.append( self.replace_prefix(test.job.script_filename, test.outputdir)) ret.extend( [self.replace_prefix(f, test.outputdir) for f in test.keep_files]) return ret def test_environ_setup(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] test.modules = ['testmod_foo'] test.variables = {'_FOO_': '1', '_BAR_': '2'} test.local = True test.setup(self.partition, self.progenv) for k in test.variables.keys(): self.assertNotIn(k, os.environ) # Manually unload the environment self.progenv.unload() def _run_test(self, test, compile_only=False): test.setup(self.partition, self.progenv) test.compile() test.compile_wait() test.run() test.wait() test.check_sanity() test.check_performance() test.cleanup(remove_files=True) self.assertFalse(os.path.exists(test.stagedir)) for f in self.keep_files_list(test, compile_only): self.assertTrue(os.path.exists(f)) @fixtures.switch_to_user_runtime def test_hellocheck(self): self.setup_remote_execution() test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] self._run_test(test) @fixtures.switch_to_user_runtime def test_hellocheck_make(self): self.setup_remote_execution() test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck_make.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] self._run_test(test) def test_hellocheck_local(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] # Test also the prebuild/postbuild functionality test.prebuild_cmd = ['touch prebuild', 'mkdir prebuild_dir'] test.postbuild_cmd = ['touch postbuild', 'mkdir postbuild_dir'] test.keep_files = [ 'prebuild', 'postbuild', 'prebuild_dir', 'postbuild_dir' ] # Force local execution of the test test.local = True self._run_test(test) def test_hellocheck_local_prepost_run(self): @sn.sanity_function def stagedir(test): return test.stagedir test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] # Test also the prebuild/postbuild functionality test.pre_run = ['echo prerun: `pwd`'] test.post_run = ['echo postrun: `pwd`'] pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1) post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1) test.sanity_patterns = sn.all([ sn.assert_eq(stagedir(test), pre_run_path), sn.assert_eq(stagedir(test), post_run_path), ]) # Force local execution of the test test.local = True self._run_test(test) def test_hellocheck_local_prepost_run_in_setup(self): def custom_setup(obj, partition, environ, **job_opts): super(obj.__class__, obj).setup(partition, environ, **job_opts) obj.pre_run = ['echo Prerunning cmd from setup phase'] obj.post_run = ['echo Postruning cmd from setup phase'] test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] # Monkey patch the setup method of the test test.setup = custom_setup.__get__(test) # Use test environment for the regression check test.valid_prog_environs = ['*'] test.sanity_patterns = sn.all([ sn.assert_found(r'^Prerunning cmd from setup phase', test.stdout), sn.assert_found(r'Hello, World\!', test.stdout), sn.assert_found(r'^Postruning cmd from setup phase', test.stdout) ]) # Force local execution of the test test.local = True self._run_test(test) def test_run_only_sanity(self): test = RunOnlyRegressionTest('runonlycheck', 'unittests/resources/checks') test.executable = './hello.sh' test.executable_opts = ['Hello, World!'] test.local = True test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout) self._run_test(test) def test_compile_only_failure(self): test = CompileOnlyRegressionTest('compileonlycheck', 'unittests/resources/checks') test.sourcepath = 'compiler_failure.c' test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.setup(self.partition, self.progenv) test.compile() self.assertRaises(BuildError, test.compile_wait) def test_compile_only_warning(self): test = CompileOnlyRegressionTest('compileonlycheckwarning', 'unittests/resources/checks') test.build_system = 'SingleSource' test.build_system.srcfile = 'compiler_warning.c' test.build_system.cflags = ['-Wall'] test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.sanity_patterns = sn.assert_found(r'warning', test.stderr) self._run_test(test, compile_only=True) @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def test_supports_system(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] test.valid_systems = ['*'] self.assertTrue(test.supports_system('gpu')) self.assertTrue(test.supports_system('login')) self.assertTrue(test.supports_system('testsys:gpu')) self.assertTrue(test.supports_system('testsys:login')) test.valid_systems = ['testsys'] self.assertTrue(test.supports_system('gpu')) self.assertTrue(test.supports_system('login')) self.assertTrue(test.supports_system('testsys:gpu')) self.assertTrue(test.supports_system('testsys:login')) test.valid_systems = ['testsys:gpu'] self.assertTrue(test.supports_system('gpu')) self.assertFalse(test.supports_system('login')) self.assertTrue(test.supports_system('testsys:gpu')) self.assertFalse(test.supports_system('testsys:login')) test.valid_systems = ['testsys:login'] self.assertFalse(test.supports_system('gpu')) self.assertTrue(test.supports_system('login')) self.assertFalse(test.supports_system('testsys:gpu')) self.assertTrue(test.supports_system('testsys:login')) test.valid_systems = ['foo'] self.assertFalse(test.supports_system('gpu')) self.assertFalse(test.supports_system('login')) self.assertFalse(test.supports_system('testsys:gpu')) self.assertFalse(test.supports_system('testsys:login')) def test_supports_environ(self): test = self.loader.load_from_file( 'unittests/resources/checks/hellocheck.py')[0] test.valid_prog_environs = ['*'] self.assertTrue(test.supports_environ('foo1')) self.assertTrue(test.supports_environ('foo-env')) self.assertTrue(test.supports_environ('*')) test.valid_prog_environs = ['PrgEnv-foo-*'] self.assertTrue(test.supports_environ('PrgEnv-foo-version1')) self.assertTrue(test.supports_environ('PrgEnv-foo-version2')) self.assertFalse(test.supports_environ('PrgEnv-boo-version1')) self.assertFalse(test.supports_environ('Prgenv-foo-version1')) def test_sourcesdir_none(self): test = RegressionTest('hellocheck', 'unittests/resources/checks') test.sourcesdir = None test.valid_prog_environs = ['*'] test.valid_systems = ['*'] self.assertRaises(ReframeError, self._run_test, test) def test_sourcesdir_none_generated_sources(self): test = RegressionTest('hellocheck_generated_sources', 'unittests/resources/checks') test.sourcesdir = None test.prebuild_cmd = [ "printf '#include <stdio.h>\\n int main(){ " "printf(\"Hello, World!\\\\n\"); return 0; }' " "> hello.c" ] test.executable = './hello' test.sourcepath = 'hello.c' test.local = True test.valid_systems = ['*'] test.valid_prog_environs = ['*'] test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout) self._run_test(test) def test_sourcesdir_none_compile_only(self): test = CompileOnlyRegressionTest('hellocheck', 'unittests/resources/checks') test.sourcesdir = None test.valid_prog_environs = ['*'] test.valid_systems = ['*'] self.assertRaises(BuildError, self._run_test, test) def test_sourcesdir_none_run_only(self): test = RunOnlyRegressionTest('hellocheck', 'unittests/resources/checks') test.sourcesdir = None test.executable = 'echo' test.executable_opts = ["Hello, World!"] test.local = True test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout) self._run_test(test) def test_sourcepath_abs(self): test = CompileOnlyRegressionTest('compileonlycheck', 'unittests/resources/checks') test.valid_prog_environs = [self.progenv.name] test.valid_systems = ['*'] test.setup(self.partition, self.progenv) test.sourcepath = '/usr/src' self.assertRaises(PipelineError, test.compile) def test_sourcepath_upref(self): test = CompileOnlyRegressionTest('compileonlycheck', 'unittests/resources/checks') test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.setup(self.partition, self.progenv) test.sourcepath = '../hellosrc' self.assertRaises(PipelineError, test.compile) @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys') def test_extra_resources(self): # Load test site configuration test = RegressionTest('dummycheck', 'unittests/resources/checks') test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.extra_resources = { 'gpu': { 'num_gpus_per_node': 2 }, 'datawarp': { 'capacity': '100GB', 'stagein_src': '/foo' } } partition = rt.runtime().system.partition('gpu') environ = partition.environment('builtin-gcc') test.setup(partition, environ) test.job.options += ['--foo'] expected_job_options = [ '--gres=gpu:2', '#DW jobdw capacity=100GB', '#DW stage_in source=/foo', '--foo' ] self.assertCountEqual(expected_job_options, test.job.options)
class TestRegressionTest(unittest.TestCase): def setUp(self): get_modules_system().searchpath_add(fixtures.TEST_MODULES) # Load a system configuration self.system, self.partition, self.progenv = fixtures.get_test_config() self.resourcesdir = tempfile.mkdtemp(dir='unittests') self.loader = RegressionCheckLoader(['unittests/resources']) self.resources = ResourcesManager(prefix=self.resourcesdir) def tearDown(self): shutil.rmtree(self.resourcesdir, ignore_errors=True) def setup_from_site(self): self.partition = fixtures.partition_with_scheduler(None) # pick the first environment of partition if self.partition.environs: self.progenv = self.partition.environs[0] def replace_prefix(self, filename, new_prefix): basename = os.path.basename(filename) return os.path.join(new_prefix, basename) def keep_files_list(self, test, compile_only=False): from reframe.core.deferrable import evaluate ret = [ self.replace_prefix(evaluate(test.stdout), test.outputdir), self.replace_prefix(evaluate(test.stderr), test.outputdir) ] if not compile_only: ret.append( self.replace_prefix(test.job.script_filename, test.outputdir)) ret.extend( [self.replace_prefix(f, test.outputdir) for f in test.keep_files]) return ret def test_environ_setup(self): test = self.loader.load_from_file('unittests/resources/hellocheck.py', system=self.system, resources=self.resources)[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] test.modules = ['testmod_foo'] test.variables = {'_FOO_': '1', '_BAR_': '2'} test.local = True test.setup(self.partition, self.progenv) for m in test.modules: self.assertTrue(get_modules_system().is_module_loaded(m)) for k, v in test.variables.items(): self.assertEqual(os.environ[k], v) # Manually unload the environment self.progenv.unload() def _run_test(self, test, compile_only=False): test.setup(self.partition, self.progenv) test.compile() test.run() test.wait() test.check_sanity() test.check_performance() test.cleanup(remove_files=True) self.assertFalse(os.path.exists(test.stagedir)) for f in self.keep_files_list(test, compile_only): self.assertTrue(os.path.exists(f)) @unittest.skipIf(not fixtures.partition_with_scheduler(None), 'job submission not supported') def test_hellocheck(self): self.setup_from_site() test = self.loader.load_from_file('unittests/resources/hellocheck.py', system=self.system, resources=self.resources)[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] self._run_test(test) @unittest.skipIf(not fixtures.partition_with_scheduler(None), 'job submission not supported') def test_hellocheck_make(self): self.setup_from_site() test = self.loader.load_from_file( 'unittests/resources/hellocheck_make.py', system=self.system, resources=self.resources)[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] self._run_test(test) def test_hellocheck_local(self): test = self.loader.load_from_file('unittests/resources/hellocheck.py', system=self.system, resources=self.resources)[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] # Test also the prebuild/postbuild functionality test.prebuild_cmd = ['touch prebuild'] test.postbuild_cmd = ['touch postbuild'] test.keepfiles = ['prebuild', 'postbuild'] # Force local execution of the test test.local = True self._run_test(test) def test_hellocheck_local_slashes(self): # Try to fool path creation by adding slashes to environment partitions # names from reframe.core.environments import ProgEnvironment self.progenv = ProgEnvironment('bad/name', self.progenv.modules, self.progenv.variables) # That's a bit hacky, but we are in a unit test self.system._name += os.sep + 'bad' self.partition._name += os.sep + 'bad' self.test_hellocheck_local() def test_hellocheck_local_prepost_run(self): @sn.sanity_function def stagedir(test): return test.stagedir test = self.loader.load_from_file('unittests/resources/hellocheck.py', system=self.system, resources=self.resources)[0] # Use test environment for the regression check test.valid_prog_environs = [self.progenv.name] # Test also the prebuild/postbuild functionality test.pre_run = ['echo prerun: `pwd`'] test.post_run = ['echo postrun: `pwd`'] pre_run_path = sn.extractsingle(r'^prerun: (\S+)', test.stdout, 1) post_run_path = sn.extractsingle(r'^postrun: (\S+)', test.stdout, 1) test.sanity_patterns = sn.all([ sn.assert_eq(stagedir(test), pre_run_path), sn.assert_eq(stagedir(test), post_run_path), ]) # Force local execution of the test test.local = True self._run_test(test) def test_run_only_sanity(self): test = RunOnlyRegressionTest('runonlycheck', 'unittests/resources', resources=self.resources, system=self.system) test.executable = './hello.sh' test.executable_opts = ['Hello, World!'] test.local = True test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout) self._run_test(test) def test_compile_only_failure(self): test = CompileOnlyRegressionTest('compileonlycheck', 'unittests/resources', resources=self.resources, system=self.system) test.sourcepath = 'compiler_failure.c' test.valid_prog_environs = [self.progenv.name] test.valid_systems = [self.system.name] test.setup(self.partition, self.progenv) self.assertRaises(CompilationError, test.compile) def test_compile_only_warning(self): test = CompileOnlyRegressionTest('compileonlycheckwarning', 'unittests/resources', resources=self.resources, system=self.system) test.sourcepath = 'compiler_warning.c' self.progenv.cflags = '-Wall' test.valid_prog_environs = [self.progenv.name] test.valid_systems = [self.system.name] test.sanity_patterns = sn.assert_found(r'warning', test.stderr) self._run_test(test, compile_only=True) def test_supports_system(self): test = self.loader.load_from_file('unittests/resources/hellocheck.py', system=self.system, resources=self.resources)[0] test.valid_systems = ['*'] self.assertTrue(test.supports_system('gpu')) self.assertTrue(test.supports_system('login')) self.assertTrue(test.supports_system('testsys:gpu')) self.assertTrue(test.supports_system('testsys:login')) test.valid_systems = ['testsys'] self.assertTrue(test.supports_system('gpu')) self.assertTrue(test.supports_system('login')) self.assertTrue(test.supports_system('testsys:gpu')) self.assertTrue(test.supports_system('testsys:login')) test.valid_systems = ['testsys:gpu'] self.assertTrue(test.supports_system('gpu')) self.assertFalse(test.supports_system('login')) self.assertTrue(test.supports_system('testsys:gpu')) self.assertFalse(test.supports_system('testsys:login')) test.valid_systems = ['testsys:login'] self.assertFalse(test.supports_system('gpu')) self.assertTrue(test.supports_system('login')) self.assertFalse(test.supports_system('testsys:gpu')) self.assertTrue(test.supports_system('testsys:login')) test.valid_systems = ['foo'] self.assertFalse(test.supports_system('gpu')) self.assertFalse(test.supports_system('login')) self.assertFalse(test.supports_system('testsys:gpu')) self.assertFalse(test.supports_system('testsys:login')) def test_supports_environ(self): test = self.loader.load_from_file('unittests/resources/hellocheck.py', system=self.system, resources=self.resources)[0] test.valid_prog_environs = ['*'] self.assertTrue(test.supports_environ('foo1')) self.assertTrue(test.supports_environ('foo-env')) self.assertTrue(test.supports_environ('*')) test.valid_prog_environs = ['PrgEnv-foo-*'] self.assertTrue(test.supports_environ('PrgEnv-foo-version1')) self.assertTrue(test.supports_environ('PrgEnv-foo-version2')) self.assertFalse(test.supports_environ('PrgEnv-boo-version1')) self.assertFalse(test.supports_environ('Prgenv-foo-version1')) def test_sourcesdir_none(self): test = RegressionTest('hellocheck', 'unittests/resources', resources=self.resources, system=self.system) test.sourcesdir = None test.valid_prog_environs = ['*'] test.valid_systems = ['*'] self.assertRaises(ReframeError, self._run_test, test) def test_sourcesdir_none_generated_sources(self): test = RegressionTest('hellocheck_generated_sources', 'unittests/resources', resources=self.resources, system=self.system) test.sourcesdir = None test.prebuild_cmd = [ "printf '#include <stdio.h>\\n int main(){ " "printf(\"Hello, World!\\\\n\"); return 0; }' " "> hello.c" ] test.executable = './hello' test.sourcepath = 'hello.c' test.local = True test.valid_systems = ['*'] test.valid_prog_environs = ['*'] test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout) self._run_test(test) def test_sourcesdir_none_compile_only(self): test = CompileOnlyRegressionTest('hellocheck', 'unittests/resources', resources=self.resources, system=self.system) test.sourcesdir = None test.valid_prog_environs = ['*'] test.valid_systems = ['*'] self.assertRaises(CompilationError, self._run_test, test) def test_sourcesdir_none_run_only(self): test = RunOnlyRegressionTest('hellocheck', 'unittests/resources', resources=self.resources, system=self.system) test.sourcesdir = None test.executable = 'echo' test.executable_opts = ["Hello, World!"] test.local = True test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.sanity_patterns = sn.assert_found(r'Hello, World\!', test.stdout) self._run_test(test) def test_sourcepath_abs(self): test = CompileOnlyRegressionTest('compileonlycheck', 'unittests/resources', resources=self.resources, system=self.system) test.valid_prog_environs = [self.progenv.name] test.valid_systems = [self.system.name] test.setup(self.partition, self.progenv) test.sourcepath = '/usr/src' self.assertRaises(PipelineError, test.compile) def test_sourcepath_upref(self): test = CompileOnlyRegressionTest('compileonlycheck', 'unittests/resources', resources=self.resources, system=self.system) test.valid_prog_environs = [self.progenv.name] test.valid_systems = [self.system.name] test.setup(self.partition, self.progenv) test.sourcepath = '../hellosrc' self.assertRaises(PipelineError, test.compile) def test_extra_resources(self): # Load test site configuration system, partition, progenv = fixtures.get_test_config() test = RegressionTest('dummycheck', 'unittests/resources', resources=self.resources, system=self.system) test.valid_prog_environs = ['*'] test.valid_systems = ['*'] test.extra_resources = { 'gpu': { 'num_gpus_per_node': 2 }, 'datawarp': { 'capacity': '100GB', 'stagein_src': '/foo' } } test.setup(self.partition, self.progenv) test.job.options += ['--foo'] expected_job_options = [ '--gres=gpu:2', '#DW jobdw capacity=100GB', '#DW stage_in source=/foo', '--foo' ] self.assertCountEqual(expected_job_options, test.job.options)
class TestRegressionCheckLoader(unittest.TestCase): def setUp(self): self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True) self.loader_with_path = RegressionCheckLoader( ['unittests/resources/checks', 'unittests/foobar'], ignore_conflicts=True) self.loader_with_prefix = RegressionCheckLoader( load_path=['bad'], prefix=os.path.abspath('unittests/resources/checks')) def test_load_file_relative(self): checks = self.loader.load_from_file( 'unittests/resources/checks/emptycheck.py') self.assertEqual(1, len(checks)) self.assertEqual(checks[0].name, 'EmptyTest') def test_load_file_absolute(self): checks = self.loader.load_from_file( os.path.abspath('unittests/resources/checks/emptycheck.py')) self.assertEqual(1, len(checks)) self.assertEqual(checks[0].name, 'EmptyTest') def test_load_recursive(self): checks = self.loader.load_from_dir('unittests/resources/checks', recurse=True) self.assertEqual(11, len(checks)) def test_load_all(self): checks = self.loader_with_path.load_all() self.assertEqual(10, len(checks)) def test_load_all_with_prefix(self): checks = self.loader_with_prefix.load_all() self.assertEqual(1, len(checks)) def test_load_new_syntax(self): checks = self.loader.load_from_file( 'unittests/resources/checks_unlisted/good.py') self.assertEqual(13, len(checks)) def test_load_mixed_syntax(self): self.assertRaises(RegressionTestLoadError, self.loader.load_from_file, 'unittests/resources/checks_unlisted/mixed.py') def test_conflicted_checks(self): self.loader_with_path._ignore_conflicts = False self.assertRaises(NameConflictError, self.loader_with_path.load_all) def test_load_error(self): self.assertRaises(OSError, self.loader.load_from_file, 'unittests/resources/checks/foo.py') def test_load_bad_required_version(self): with self.assertRaises(ValueError): self.loader.load_from_file('unittests/resources/checks_unlisted/' 'no_required_version.py') def test_load_bad_init(self): tests = self.loader.load_from_file( 'unittests/resources/checks_unlisted/bad_init_check.py') self.assertEqual(0, len(tests))
def load_test(testfile): loader = RegressionCheckLoader(['unittests/resources/checks']) return loader.load_from_file(testfile)
class TestRegressionCheckLoader(unittest.TestCase): def setUp(self): self.loader = RegressionCheckLoader(['.'], ignore_conflicts=True) self.loader_with_path = RegressionCheckLoader( ['unittests/resources/checks', 'unittests/foobar'], ignore_conflicts=True) def test_load_file_relative(self): checks = self.loader.load_from_file( 'unittests/resources/checks/emptycheck.py') assert 1 == len(checks) assert checks[0].name == 'EmptyTest' def test_load_file_absolute(self): checks = self.loader.load_from_file( os.path.abspath('unittests/resources/checks/emptycheck.py')) assert 1 == len(checks) assert checks[0].name == 'EmptyTest' def test_load_recursive(self): checks = self.loader.load_from_dir('unittests/resources/checks', recurse=True) assert 12 == len(checks) def test_load_all(self): checks = self.loader_with_path.load_all() assert 11 == len(checks) def test_load_new_syntax(self): checks = self.loader.load_from_file( 'unittests/resources/checks_unlisted/good.py') assert 13 == len(checks) def test_conflicted_checks(self): self.loader_with_path._ignore_conflicts = False with pytest.raises(NameConflictError): self.loader_with_path.load_all() def test_load_error(self): with pytest.raises(OSError): self.loader.load_from_file('unittests/resources/checks/foo.py') def test_load_bad_required_version(self): with pytest.raises(ValueError): self.loader.load_from_file('unittests/resources/checks_unlisted/' 'no_required_version.py') def test_load_bad_init(self): tests = self.loader.load_from_file( 'unittests/resources/checks_unlisted/bad_init_check.py') assert 0 == len(tests) def test_special_test(self): with pytest.warns(ReframeDeprecationWarning): @rfm.simple_test class TestDeprecated(rfm.RegressionTest): def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) with pytest.warns(ReframeDeprecationWarning): @rfm.simple_test class TestDeprecatedRunOnly(rfm.RunOnlyRegressionTest): def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) with pytest.warns(ReframeDeprecationWarning): @rfm.simple_test class TestDeprecatedCompileOnly(rfm.CompileOnlyRegressionTest): def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) with pytest.warns(ReframeDeprecationWarning): @rfm.simple_test class TestDeprecatedCompileOnlyDerived(TestDeprecatedCompileOnly): def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) with pytest.warns(None) as warnings: @rfm.simple_test class TestSimple(rfm.RegressionTest): def __init__(self): pass @rfm.simple_test class TestSpecial(rfm.RegressionTest, special=True): def __init__(self): pass def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) @rfm.simple_test class TestSpecialRunOnly(rfm.RunOnlyRegressionTest, special=True): def __init__(self): pass def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) def run(self): super().run() @rfm.simple_test class TestSpecialCompileOnly(rfm.CompileOnlyRegressionTest, special=True): def __init__(self): pass def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) def run(self): super().run() assert not any( isinstance(w.message, ReframeDeprecationWarning) for w in warnings) with pytest.warns(ReframeDeprecationWarning) as warnings: @rfm.simple_test class TestSpecialDerived(TestSpecial): def __init__(self): pass def setup(self, partition, environ, **job_opts): super().setup(system, environ, **job_opts) def run(self): super().run() assert len(warnings) == 2 @rfm.simple_test class TestFinal(rfm.RegressionTest): def __init__(self): pass @rfm.final def my_new_final(seld): pass with pytest.warns(ReframeDeprecationWarning): @rfm.simple_test class TestFinalDerived(TestFinal): def my_new_final(self, a, b): pass