def test_toposort_subgraph(self): # # t0 # ^ # | # +-->t1<--+ # | | # t2<------t3 # ^ ^ # | | # +---t4---+ # t0 = self.create_test('t0') t1 = self.create_test('t1') t2 = self.create_test('t2') t3 = self.create_test('t3') t4 = self.create_test('t4') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') full_deps = dependency.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4])) partial_deps = dependency.build_deps( executors.generate_testcases([t3, t4]), full_deps) cases = dependency.toposort(partial_deps, is_subgraph=True) self.assert_topological_order(cases, partial_deps)
def test_prune_deps(default_exec_ctx): # # t0 +-->t5<--+ # ^ | | # | | | # +-->t1<--+ t6 t7 # | | ^ # t2<------t3 | # ^ ^ | # | | t8 # +---t4---+ # t0 = make_test('t0') t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t5 = make_test('t5') t6 = make_test('t6') t7 = make_test('t7') t8 = make_test('t8') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') t6.depends_on('t5') t7.depends_on('t5') t8.depends_on('t7') testcases_all = executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8]) testcases = executors.generate_testcases([t3, t7]) full_deps, _ = dependencies.build_deps(testcases_all) pruned_deps = dependencies.prune_deps(full_deps, testcases) # Check the connectivity assert len(pruned_deps) == 6*4 for p in ['sys0:p0', 'sys0:p1']: for e in ['e0', 'e1']: node = functools.partial(Node, pname=p, ename=e) assert has_edge(pruned_deps, node('t3'), node('t2')) assert has_edge(pruned_deps, node('t3'), node('t1')) assert has_edge(pruned_deps, node('t2'), node('t1')) assert has_edge(pruned_deps, node('t1'), node('t0')) assert has_edge(pruned_deps, node('t7'), node('t5')) assert len(pruned_deps[node('t3')]) == 2 assert len(pruned_deps[node('t2')]) == 1 assert len(pruned_deps[node('t1')]) == 1 assert len(pruned_deps[node('t7')]) == 1 assert len(pruned_deps[node('t5')]) == 0 assert len(pruned_deps[node('t0')]) == 0
def repeat_tests(testcases, num_repeats): '''Returns new test cases parameterized over their repetition number''' tmp_registry = TestRegistry() unique_checks = set() for tc in testcases: check = tc.check if check.is_fixture() or check in unique_checks: continue unique_checks.add(check) cls = type(check) variant_info = cls.get_variant_info( check.variant_num, recurse=True ) nc = make_test( f'{cls.__name__}', (cls,), { '$repeat_no': builtins.parameter(range(num_repeats)) } ) nc._rfm_custom_prefix = check.prefix for i in range(nc.num_variants): # Check if this variant should be instantiated vinfo = nc.get_variant_info(i, recurse=True) vinfo['params'].pop('$repeat_no') if vinfo == variant_info: tmp_registry.add(nc, variant_num=i) new_checks = tmp_registry.instantiate_all() return generate_testcases(new_checks)
def test_skip_unresolved_deps(make_exec_ctx): # # t0 t4 # ^ ^ ^ # / \ / # t1 t2 # ^ # | # t3 # make_exec_ctx(system='sys0:p0') t0 = make_test('t0') t0.valid_systems = ['sys0:p1'] t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t1.depends_on('t0') t2.depends_on('t0') t2.depends_on('t4') t3.depends_on('t2') deps, skipped_cases = dependencies.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4]) ) assert len(skipped_cases) == 6 skipped_tests = {c.check.name for c in skipped_cases} assert skipped_tests == {'t1', 't2', 't3'}
def _make_cases(): @fixtures.custom_prefix('unittests/resources/checks') class _T0(rfm.RegressionTest): valid_systems = ['*'] valid_prog_environs = ['*'] sourcepath = 'hello.c' executable = 'echo' sanity_patterns = sn.assert_true(1) def check_and_skip(self): self.skip_if(True) # Attach the hook manually based on the request.param when, stage = request.param.split('_', maxsplit=1) hook = rfm.run_before if when == 'pre' else rfm.run_after check_and_skip = hook(stage)(check_and_skip) class _T1(rfm.RunOnlyRegressionTest): valid_systems = ['*'] valid_prog_environs = ['*'] sanity_patterns = sn.assert_true(1) def __init__(self): self.depends_on(_T0.__qualname__) cases = executors.generate_testcases([_T0(), _T1()]) depgraph, _ = dependencies.build_deps(cases) return dependencies.toposort(depgraph), request.param
def test_valid_deps(make_test, default_exec_ctx): # # t0 +-->t5<--+ # ^ | | # | | | # +-->t1<--+ t6 t7 # | | ^ # t2<------t3 | # ^ ^ | # | | t8 # +---t4---+ # t0 = make_test('t0') t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t5 = make_test('t5') t6 = make_test('t6') t7 = make_test('t7') t8 = make_test('t8') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') t6.depends_on('t5') t7.depends_on('t5') t8.depends_on('t7') dependencies.validate_deps( dependencies.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8]))[0])
def test_valid_deps(self): # # t0 +-->t5<--+ # ^ | | # | | | # +-->t1<--+ t6 t7 # | | ^ # t2<------t3 | # ^ ^ | # | | t8 # +---t4---+ # t0 = self.create_test('t0') t1 = self.create_test('t1') t2 = self.create_test('t2') t3 = self.create_test('t3') t4 = self.create_test('t4') t5 = self.create_test('t5') t6 = self.create_test('t6') t7 = self.create_test('t7') t8 = self.create_test('t8') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') t6.depends_on('t5') t7.depends_on('t5') t8.depends_on('t7') dependency.validate_deps( dependency.build_deps( executors.generate_testcases( [t0, t1, t2, t3, t4, t5, t6, t7, t8])))
def test_config_params(make_runner, make_exec_ctx): '''Test that configuration parameters are properly retrieved with the various execution policies. ''' class T(rfm.RunOnlyRegressionTest): valid_systems = ['generic2'] valid_prog_environs = ['*'] executable = 'echo' @sanity_function def validate(self): return True @run_after('setup') def assert_git_timeout(self): expected = 10 if self.current_partition.name == 'part1' else 20 timeout = rt.runtime().get_option('general/0/git_timeout') assert timeout == expected make_exec_ctx(system='generic2') runner = make_runner() testcases = executors.generate_testcases([T()]) runner.runall(testcases) assert runner.stats.num_cases() == 2 assert not runner.stats.failed()
def test_require_deps(HelloTest, local_exec_ctx): import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors @test_util.custom_prefix('unittests/resources/checks') class T0(HelloTest): x = variable(int, value=1) @test_util.custom_prefix('unittests/resources/checks') class T1(HelloTest): @run_after('init') def setdeps(self): self.depends_on('T0') @require_deps def sety(self, T0): self.y = T0().x + 1 @run_before('run') @require_deps def setz(self, T0): self.z = T0().x + 2 cases = executors.generate_testcases([T0(), T1()]) deps, _ = dependencies.build_deps(cases) for c in dependencies.toposort(deps): _run(*c) for c in cases: t = c.check if t.name == 'T0': assert t.x == 1 elif t.name == 'T1': assert t.y == 2 assert t.z == 3
def test_toposort(self): # # t0 +-->t5<--+ # ^ | | # | | | # +-->t1<--+ t6 t7 # | | ^ # t2<------t3 | # ^ ^ | # | | t8 # +---t4---+ # t0 = self.create_test('t0') t1 = self.create_test('t1') t2 = self.create_test('t2') t3 = self.create_test('t3') t4 = self.create_test('t4') t5 = self.create_test('t5') t6 = self.create_test('t6') t7 = self.create_test('t7') t8 = self.create_test('t8') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') t6.depends_on('t5') t7.depends_on('t5') t8.depends_on('t7') deps = dependency.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8])) cases = dependency.toposort(deps) self.assert_topological_order(cases, deps)
def test_distribute_testcases(loader, default_exec_ctx): testcases = executors.generate_testcases(loader.load_all()) testcases = filter(filters.have_any_name('Simple'), testcases) testcases = list(testcases) assert len(testcases) == 4 count = sum(map(lambda x: x.partition.fullname == 'sys0:p0', testcases)) assert count == 2 count = sum(map(lambda x: x.partition.fullname == 'sys0:p1', testcases)) assert count == 2 node_map = {'sys0:p0': ['n1', 'n2'], 'sys0:p1': ['n3']} new_cases = distribute_tests(testcases, node_map) assert len(new_cases) == 6 count = sum(map(lambda x: x.partition.fullname == 'sys0:p0', new_cases)) assert count == 4 count = sum(map(lambda x: x.partition.fullname == 'sys0:p1', new_cases)) assert count == 2 def sys0p0_nodes(): for nodelist in (['n2'], ['n2'], ['n1'], ['n1']): yield nodelist nodelist_iter = sys0p0_nodes() for tc in new_cases: nodes = getattr(tc.check, '$nid') if tc.partition.fullname == 'sys0:p0': assert nodes == next(nodelist_iter) else: assert nodes == ['n3'] # Make sure we have consumed all the elements from nodelist_iter with pytest.raises(StopIteration): next(nodelist_iter)
def test_skip_unresolved_deps(make_test, temp_runtime): # # t0 t4 # ^ ^ ^ # / \ / # t1 t2 # ^ # | # t3 # rt = temp_runtime(fixtures.TEST_CONFIG_FILE, system='sys0:p0') next(rt) t0 = make_test('t0') t0.valid_systems = ['sys0:p1'] t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t1.depends_on('t0') t2.depends_on('t0') t2.depends_on('t4') t3.depends_on('t2') deps, skipped_cases = dependencies.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4]) ) assert len(skipped_cases) == 6 skipped_tests = {c.check.name for c in skipped_cases} assert skipped_tests == {'t1', 't2', 't3'}
def runall(self, checks, sort=False, *args, **kwargs): cases = executors.generate_testcases(checks, *args, **kwargs) if sort: depgraph = dependency.build_deps(cases) dependency.validate_deps(depgraph) cases = dependency.toposort(depgraph) self.runner.runall(cases)
def test_build_deps_unknown_target_env(loader, exec_ctx): checks = loader.load_all() # Add some inexistent dependencies test0 = find_check('Test0', checks) test1 = find_check('Test1_default', checks) test1.depends_on('Test0', rfm.DEPEND_EXACT, {'e0': ['eX']}) with pytest.raises(DependencyError): dependency.build_deps(executors.generate_testcases(checks))
def test_eq_hash(loader, default_exec_ctx): cases = executors.generate_testcases(loader.load_all()) case0 = find_case('Test0', 'e0', 'p0', cases) case1 = find_case('Test0', 'e1', 'p0', cases) case0_copy = case0.clone() assert case0 == case0_copy assert hash(case0) == hash(case0_copy) assert case1 != case0 assert hash(case1) != hash(case0)
def _make_cases(checks=None, sort=False, *args, **kwargs): if checks is None: checks = make_loader(['unittests/resources/checks']).load_all() cases = executors.generate_testcases(checks, *args, **kwargs) if sort: depgraph, _ = dependencies.build_deps(cases) dependencies.validate_deps(depgraph) cases = dependencies.toposort(depgraph) return cases
def test_cyclic_deps_by_env(self): t0 = self.create_test('t0') t1 = self.create_test('t1') t1.depends_on('t0', rfm.DEPEND_EXACT, {'e0': ['e0']}) t0.depends_on('t1', rfm.DEPEND_EXACT, {'e1': ['e1']}) deps = dependency.build_deps(executors.generate_testcases([t0, t1])) with pytest.raises(DependencyError) as exc_info: dependency.validate_deps(deps) assert ('t1->t0->t1' in str(exc_info.value) or 't0->t1->t0' in str(exc_info.value))
def test_cyclic_deps_by_env(make_test, default_exec_ctx): t0 = make_test('t0') t1 = make_test('t1') t1.depends_on('t0', udeps.env_is('e0')) t0.depends_on('t1', udeps.env_is('e1')) deps, _ = dependencies.build_deps(executors.generate_testcases([t0, t1])) with pytest.raises(DependencyError) as exc_info: dependencies.validate_deps(deps) assert ('t1->t0->t1' in str(exc_info.value) or 't0->t1->t0' in str(exc_info.value))
def test_build_deps_unknown_source_env(loader, exec_ctx): checks = loader.load_all() # Add some inexistent dependencies test0 = find_check('Test0', checks) test1 = find_check('Test1_default', checks) test1.depends_on('Test0', rfm.DEPEND_EXACT, {'eX': ['e0']}) # Unknown source is ignored, because it might simply be that the test # is not executed for eX deps = dependency.build_deps(executors.generate_testcases(checks)) assert num_deps(deps, 'Test1_default') == 4
def test_eq_hash(self): find_case = TestDependencies.find_case cases = executors.generate_testcases(self.loader.load_all()) case0 = find_case('Test0', 'e0', cases) case1 = find_case('Test0', 'e1', cases) case0_copy = case0.clone() assert case0 == case0_copy assert hash(case0) == hash(case0_copy) assert case1 != case0 assert hash(case1) != hash(case0)
def test_toposort_subgraph(default_exec_ctx): # # t0 # ^ # | # +-->t1<--+ # | | # t2<------t3 # ^ ^ # | | # +---t4---+ # t0 = make_test('t0') t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') full_deps, _ = dependencies.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4]) ) partial_deps, _ = dependencies.build_deps( executors.generate_testcases([t3, t4]), full_deps ) cases = dependencies.toposort(partial_deps, is_subgraph=True) assert_topological_order(cases, partial_deps) # Assert the level assignment cases_by_level = {} for c in cases: cases_by_level.setdefault(c.level, set()) cases_by_level[c.level].add(c.check.name) assert cases_by_level[1] == {'t3'} assert cases_by_level[2] == {'t4'}
def distribute_tests(testcases, node_map): '''Returns new testcases that will be parameterized to run in node of their partitions based on the nodemap ''' tmp_registry = TestRegistry() # We don't want to register the same check for every environment # per partition check_part_combs = set() for tc in testcases: check, partition, _ = tc candidate_comb = (check.unique_name, partition.fullname) if check.is_fixture() or candidate_comb in check_part_combs: continue check_part_combs.add(candidate_comb) cls = type(check) variant_info = cls.get_variant_info( check.variant_num, recurse=True ) nc = make_test( f'{cls.__name__}_{partition.fullname.replace(":", "_")}', (cls,), { 'valid_systems': [partition.fullname], '$nid': builtins.parameter( [[n] for n in node_map[partition.fullname]], fmt=util.nodelist_abbrev ) }, methods=[ builtins.run_before('run')(_rfm_pin_run_nodes), builtins.run_before('compile')(_rfm_pin_build_nodes), # We re-set the valid system in a hook to make sure that it # will not be overwriten by a parent post-init hook builtins.run_after('init')( make_valid_systems_hook([partition.fullname]) ), ] ) # We have to set the prefix manually nc._rfm_custom_prefix = check.prefix for i in range(nc.num_variants): # Check if this variant should be instantiated vinfo = nc.get_variant_info(i, recurse=True) vinfo['params'].pop('$nid') if vinfo == variant_info: tmp_registry.add(nc, variant_num=i) new_checks = tmp_registry.instantiate_all() return generate_testcases(new_checks)
def test_toposort(default_exec_ctx): # # t0 +-->t5<--+ # ^ | | # | | | # +-->t1<--+ t6 t7 # | | ^ # t2<------t3 | # ^ ^ | # | | t8 # +---t4---+ # t0 = make_test('t0') t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t5 = make_test('t5') t6 = make_test('t6') t7 = make_test('t7') t8 = make_test('t8') t1.depends_on('t0') t2.depends_on('t1') t3.depends_on('t1') t3.depends_on('t2') t4.depends_on('t2') t4.depends_on('t3') t6.depends_on('t5') t7.depends_on('t5') t8.depends_on('t7') deps, _ = dependencies.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8]) ) cases = dependencies.toposort(deps) assert_topological_order(cases, deps) # Assert the level assignment cases_by_level = {} for c in cases: cases_by_level.setdefault(c.level, set()) cases_by_level[c.level].add(c.check.name) assert cases_by_level[0] == {'t0', 't5'} assert cases_by_level[1] == {'t1', 't6', 't7'} assert cases_by_level[2] == {'t2', 't8'} assert cases_by_level[3] == {'t3'} assert cases_by_level[4] == {'t4'}
def test_ci_gitlab_pipeline(): loader = RegressionCheckLoader( ['unittests/resources/checks_unlisted/deps_complex.py']) cases = dependencies.toposort( dependencies.build_deps(executors.generate_testcases( loader.load_all()))[0]) with io.StringIO() as fp: ci.emit_pipeline(fp, cases) pipeline = fp.getvalue() # Fetch the latest Gitlab CI JSON schema response = requests.get('https://json.schemastore.org/gitlab-ci') assert response.ok schema = response.json() jsonschema.validate(yaml.safe_load(pipeline), schema)
def test_build_deps_unknown_test(loader, exec_ctx): checks = loader.load_all() # Add some inexistent dependencies test0 = find_check('Test0', checks) for depkind in ('default', 'fully', 'by_env', 'exact'): test1 = find_check('Test1_' + depkind, checks) if depkind == 'default': test1.depends_on('TestX') elif depkind == 'exact': test1.depends_on('TestX', rfm.DEPEND_EXACT, {'e0': ['e0']}) elif depkind == 'fully': test1.depends_on('TestX', rfm.DEPEND_FULLY) elif depkind == 'by_env': test1.depends_on('TestX', rfm.DEPEND_BY_ENV) with pytest.raises(DependencyError): dependency.build_deps(executors.generate_testcases(checks))
def test_cyclic_deps(default_exec_ctx): # # t0 +-->t5<--+ # ^ | | # | | | # +-->t1<--+ t6 t7 # | | | ^ # t2 | t3 | # ^ | ^ | # | v | t8 # +---t4---+ # t0 = make_test('t0') t1 = make_test('t1') t2 = make_test('t2') t3 = make_test('t3') t4 = make_test('t4') t5 = make_test('t5') t6 = make_test('t6') t7 = make_test('t7') t8 = make_test('t8') t1.depends_on('t0') t1.depends_on('t4') t2.depends_on('t1') t3.depends_on('t1') t4.depends_on('t2') t4.depends_on('t3') t6.depends_on('t5') t7.depends_on('t5') t8.depends_on('t7') deps, _ = dependencies.build_deps( executors.generate_testcases([t0, t1, t2, t3, t4, t5, t6, t7, t8]) ) with pytest.raises(DependencyError) as exc_info: dependencies.validate_deps(deps) assert ('t4->t2->t1->t4' in str(exc_info.value) or 't2->t1->t4->t2' in str(exc_info.value) or 't1->t4->t2->t1' in str(exc_info.value) or 't1->t4->t3->t1' in str(exc_info.value) or 't4->t3->t1->t4' in str(exc_info.value) or 't3->t1->t4->t3' in str(exc_info.value))
def test_require_deps(self): import unittests.resources.checks.hellocheck as mod import reframe.frontend.dependency as dependency import reframe.frontend.executors as executors class T0(mod.HelloTest): def __init__(self): super().__init__() self._prefix = 'unittests/resources/checks' self.name = type(self).__name__ self.executable = os.path.join('.', self.name) self.x = 1 class T1(mod.HelloTest): def __init__(self): super().__init__() self._prefix = 'unittests/resources/checks' self.name = type(self).__name__ self.executable = os.path.join('.', self.name) self.depends_on('T0') @rfm.require_deps def sety(self, T0): self.y = T0().x + 1 @rfm.run_before('run') @rfm.require_deps def setz(self, T0): self.z = T0().x + 2 cases = executors.generate_testcases([T0(), T1()]) deps = dependency.build_deps(cases) for c in dependency.toposort(deps): _run(*c) for c in cases: t = c.check if t.name == 'T0': assert t.x == 1 elif t.name == 'T1': assert t.y == 2 assert t.z == 3
def test_require_deps(HelloTest, local_exec_ctx): import reframe.frontend.dependencies as dependencies import reframe.frontend.executors as executors @test_util.custom_prefix('unittests/resources/checks') class T0(HelloTest): def __init__(self): super().__init__() self.name = type(self).__name__ self.executable = os.path.join('.', self.name) self.x = 1 @test_util.custom_prefix('unittests/resources/checks') class T1(HelloTest): def __init__(self): super().__init__() self.name = type(self).__name__ self.executable = os.path.join('.', self.name) self.depends_on('T0') @require_deps def sety(self, T0): self.y = T0().x + 1 @run_before('run') @require_deps def setz(self, T0): self.z = T0().x + 2 cases = executors.generate_testcases([T0(), T1()]) deps, _ = dependencies.build_deps(cases) for c in dependencies.toposort(deps): _run(*c) for c in cases: t = c.check if t.name == 'T0': assert t.x == 1 elif t.name == 'T1': assert t.y == 2 assert t.z == 3
def test_ci_gitlab_pipeline(): loader = RegressionCheckLoader( ['unittests/resources/checks_unlisted/deps_complex.py']) cases = dependencies.toposort( dependencies.build_deps(executors.generate_testcases( loader.load_all()))[0]) with io.StringIO() as fp: ci.emit_pipeline(fp, cases) pipeline = fp.getvalue() # Fetch the latest Gitlab CI JSON schema try: response = requests.get( 'https://gitlab.com/gitlab-org/gitlab/-/raw/master/app/assets/javascripts/editor/schema/ci.json' # noqa: E501 ) except requests.exceptions.ConnectionError as e: pytest.skip(f'could not reach URL: {e}') else: assert response.ok schema = response.json() jsonschema.validate(yaml.safe_load(pipeline), schema)
def main(): # Setup command line options argparser = argparse.ArgumentParser() output_options = argparser.add_argument_group('Options controlling output') locate_options = argparser.add_argument_group( 'Options for locating checks') select_options = argparser.add_argument_group( 'Options for selecting checks') action_options = argparser.add_argument_group( 'Options controlling actions') run_options = argparser.add_argument_group( 'Options controlling execution of checks') env_options = argparser.add_argument_group( 'Options controlling environment') misc_options = argparser.add_argument_group('Miscellaneous options') # Output directory options output_options.add_argument('--prefix', action='store', metavar='DIR', help='Set output directory prefix to DIR') output_options.add_argument('-o', '--output', action='store', metavar='DIR', help='Set output directory to DIR') output_options.add_argument('-s', '--stage', action='store', metavar='DIR', help='Set stage directory to DIR') output_options.add_argument( '--perflogdir', action='store', metavar='DIR', help='Set directory prefix for the performance logs ' '(default: ${prefix}/perflogs, ' 'relevant only if the filelog backend is used)') output_options.add_argument( '--keep-stage-files', action='store_true', help='Keep stage directory even if check is successful') output_options.add_argument( '--save-log-files', action='store_true', default=False, help='Copy the log file from the work dir to the output dir at the ' 'end of the program') # Check discovery options locate_options.add_argument('-c', '--checkpath', action='append', metavar='DIR|FILE', help='Search for checks in DIR or FILE') locate_options.add_argument('-R', '--recursive', action='store_true', help='Load checks recursively') locate_options.add_argument('--ignore-check-conflicts', action='store_true', help='Skip checks with conflicting names') # Select options select_options.add_argument('-t', '--tag', action='append', dest='tags', default=[], help='Select checks matching TAG') select_options.add_argument('-n', '--name', action='append', dest='names', default=[], metavar='NAME', help='Select checks with NAME') select_options.add_argument('-x', '--exclude', action='append', dest='exclude_names', metavar='NAME', default=[], help='Exclude checks with NAME') select_options.add_argument( '-p', '--prgenv', action='append', default=[r'.*'], help='Select tests for PRGENV programming environment only') select_options.add_argument('--gpu-only', action='store_true', help='Select only GPU tests') select_options.add_argument('--cpu-only', action='store_true', help='Select only CPU tests') # Action options action_options.add_argument('-l', '--list', action='store_true', help='List matched regression checks') action_options.add_argument( '-L', '--list-detailed', action='store_true', help='List matched regression checks with a detailed description') action_options.add_argument('-r', '--run', action='store_true', help='Run regression with the selected checks') # Run options run_options.add_argument('-A', '--account', action='store', help='Use ACCOUNT for submitting jobs') run_options.add_argument('-P', '--partition', action='store', metavar='PART', help='Use PART for submitting jobs') run_options.add_argument('--reservation', action='store', metavar='RES', help='Use RES for submitting jobs') run_options.add_argument('--nodelist', action='store', help='Run checks on the selected list of nodes') run_options.add_argument( '--exclude-nodes', action='store', metavar='NODELIST', help='Exclude the list of nodes from running checks') run_options.add_argument('--job-option', action='append', metavar='OPT', dest='job_options', default=[], help='Pass OPT to job scheduler') run_options.add_argument('--force-local', action='store_true', help='Force local execution of checks') run_options.add_argument('--skip-sanity-check', action='store_true', help='Skip sanity checking') run_options.add_argument('--skip-performance-check', action='store_true', help='Skip performance checking') run_options.add_argument('--strict', action='store_true', help='Force strict performance checking') run_options.add_argument('--skip-system-check', action='store_true', help='Skip system check') run_options.add_argument('--skip-prgenv-check', action='store_true', help='Skip prog. environment check') run_options.add_argument( '--exec-policy', metavar='POLICY', action='store', choices=['serial', 'async'], default='serial', help='Specify the execution policy for running the regression tests. ' 'Available policies: "serial" (default), "async"') run_options.add_argument('--mode', action='store', help='Execution mode to use') run_options.add_argument( '--max-retries', metavar='NUM', action='store', default=0, help='Specify the maximum number of times a failed regression test ' 'may be retried (default: 0)') run_options.add_argument( '--flex-alloc-tasks', action='store', dest='flex_alloc_tasks', metavar='{all|idle|NUM}', default='idle', help="Strategy for flexible task allocation (default: 'idle').") env_options.add_argument('-M', '--map-module', action='append', metavar='MAPPING', dest='module_mappings', default=[], help='Apply a single module mapping') env_options.add_argument( '-m', '--module', action='append', default=[], metavar='MOD', dest='user_modules', help='Load module MOD before running the regression suite') env_options.add_argument('--module-mappings', action='store', metavar='FILE', dest='module_map_file', help='Apply module mappings defined in FILE') env_options.add_argument( '-u', '--unload-module', action='append', metavar='MOD', dest='unload_modules', default=[], help='Unload module MOD before running the regression suite') env_options.add_argument( '--purge-env', action='store_true', dest='purge_env', default=False, help='Purge environment before running the regression suite') # Miscellaneous options misc_options.add_argument( '-C', '--config-file', action='store', dest='config_file', metavar='FILE', default=os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py'), help='Specify a custom config-file for the machine. ' '(default: %s' % os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py')) misc_options.add_argument('--nocolor', action='store_false', dest='colorize', default=True, help='Disable coloring of output') misc_options.add_argument('--performance-report', action='store_true', help='Print the performance report') # FIXME: This should move to env_options as soon as # https://github.com/eth-cscs/reframe/pull/946 is merged misc_options.add_argument('--non-default-craype', action='store_true', default=False, help='Test a non-default Cray PE') misc_options.add_argument( '--show-config', action='store_true', help='Print configuration of the current system and exit') misc_options.add_argument( '--show-config-env', action='store', metavar='ENV', help='Print configuration of environment ENV and exit') misc_options.add_argument('--system', action='store', help='Load SYSTEM configuration explicitly') misc_options.add_argument( '--timestamp', action='store', nargs='?', const='%FT%T', metavar='TIMEFMT', help='Append a timestamp component to the regression directories' '(default format "%%FT%%T")') misc_options.add_argument('-V', '--version', action='version', version=reframe.VERSION) misc_options.add_argument('-v', '--verbose', action='count', default=0, help='Increase verbosity level of output') if len(sys.argv) == 1: argparser.print_help() sys.exit(1) # Parse command line options = argparser.parse_args() # Load configuration try: settings = config.load_settings_from_file(options.config_file) except (OSError, ReframeError) as e: sys.stderr.write('%s: could not load settings: %s\n' % (sys.argv[0], e)) sys.exit(1) # Configure logging try: logging.configure_logging(settings.logging_config), except (OSError, ConfigError) as e: sys.stderr.write('could not configure logging: %s\n' % e) sys.exit(1) # Set colors in logger logging.getlogger().colorize = options.colorize # Setup printer printer = PrettyPrinter() printer.colorize = options.colorize if options.verbose: printer.inc_verbosity(options.verbose) try: runtime.init_runtime(settings.site_configuration, options.system, non_default_craype=options.non_default_craype) except SystemAutodetectionError: printer.warning( 'could not find a configuration entry for the current system; ' 'falling back to a generic system configuration; ' 'please check the online documentation on how to configure ' 'ReFrame for your system.') settings.site_configuration['systems'] = { 'generic': { 'descr': 'Generic fallback system configuration', 'hostnames': ['localhost'], 'partitions': { 'login': { 'scheduler': 'local', 'environs': ['builtin-gcc'], 'descr': 'Login nodes' } } } } settings.site_configuration['environments'] = { '*': { 'builtin-gcc': { 'type': 'ProgEnvironment', 'cc': 'gcc', 'cxx': 'g++', 'ftn': 'gfortran', } } } runtime.init_runtime(settings.site_configuration, 'generic', non_default_craype=options.non_default_craype) except Exception as e: printer.error('configuration error: %s' % e) printer.verbose(''.join(traceback.format_exception(*sys.exc_info()))) sys.exit(1) rt = runtime.runtime() try: if options.module_map_file: rt.modules_system.load_mapping_from_file(options.module_map_file) if options.module_mappings: for m in options.module_mappings: rt.modules_system.load_mapping(m) except (ConfigError, OSError) as e: printer.error('could not load module mappings: %s' % e) sys.exit(1) if options.mode: try: mode_args = rt.mode(options.mode) # Parse the mode's options and reparse the command-line options = argparser.parse_args(mode_args) options = argparser.parse_args(namespace=options) except ConfigError as e: printer.error('could not obtain execution mode: %s' % e) sys.exit(1) # Adjust system directories if options.prefix: # if prefix is set, reset all other directories rt.resources.prefix = os_ext.expandvars(options.prefix) rt.resources.outputdir = None rt.resources.stagedir = None if options.output: rt.resources.outputdir = os_ext.expandvars(options.output) if options.stage: rt.resources.stagedir = os_ext.expandvars(options.stage) if (os_ext.samefile(rt.resources.stage_prefix, rt.resources.output_prefix) and not options.keep_stage_files): printer.error('stage and output refer to the same directory; ' 'if this is on purpose, please use also the ' "`--keep-stage-files' option.") sys.exit(1) if options.timestamp: rt.resources.timefmt = options.timestamp # Configure performance logging # NOTE: we need resources to be configured in order to set the global # perf. logging prefix correctly if options.perflogdir: rt.resources.perflogdir = os_ext.expandvars(options.perflogdir) logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = ( rt.resources.perflog_prefix) # Show configuration after everything is set up if options.show_config: printer.info(rt.show_config()) sys.exit(0) if options.show_config_env: envname = options.show_config_env for p in rt.system.partitions: environ = p.environment(envname) if environ: break if environ is None: printer.error('no such environment: ' + envname) sys.exit(1) printer.info(environ.details()) sys.exit(0) if hasattr(settings, 'perf_logging_config'): try: logging.configure_perflogging(settings.perf_logging_config) except (OSError, ConfigError) as e: printer.error('could not configure performance logging: %s\n' % e) sys.exit(1) else: printer.warning('no performance logging is configured; ' 'please check documentation') # Setup the check loader if options.checkpath: load_path = [] for d in options.checkpath: d = os_ext.expandvars(d) if not os.path.exists(d): printer.warning("%s: path `%s' does not exist. Skipping..." % (argparser.prog, d)) continue load_path.append(d) loader = RegressionCheckLoader( load_path, recurse=options.recursive, ignore_conflicts=options.ignore_check_conflicts) else: loader = RegressionCheckLoader(load_path=settings.checks_path, prefix=reframe.INSTALL_PREFIX, recurse=settings.checks_path_recurse) printer.debug(argparse.format_options(options)) # Print command line printer.info('Command line: %s' % ' '.join(sys.argv)) printer.info('Reframe version: ' + reframe.VERSION) printer.info('Launched by user: '******'<unknown>')) printer.info('Launched on host: ' + socket.gethostname()) # Print important paths printer.info('Reframe paths') printer.info('=============') printer.info(' Check prefix : %s' % loader.prefix) printer.info( '%03s Check search path : %s' % ('(R)' if loader.recurse else '', "'%s'" % ':'.join(loader.load_path))) printer.info(' Stage dir prefix : %s' % rt.resources.stage_prefix) printer.info(' Output dir prefix : %s' % rt.resources.output_prefix) printer.info( ' Perf. logging prefix : %s' % os.path.abspath(logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'])) try: # Locate and load checks try: checks_found = loader.load_all() except OSError as e: raise ReframeError from e # Filter checks by name checks_matched = checks_found if options.exclude_names: for name in options.exclude_names: checks_matched = filter(filters.have_not_name(name), checks_matched) if options.names: checks_matched = filter(filters.have_name('|'.join(options.names)), checks_matched) # Filter checks by tags for tag in options.tags: checks_matched = filter(filters.have_tag(tag), checks_matched) # Filter checks by prgenv if not options.skip_prgenv_check: for prgenv in options.prgenv: checks_matched = filter(filters.have_prgenv(prgenv), checks_matched) # Filter checks by system if not options.skip_system_check: checks_matched = filter( filters.have_partition(rt.system.partitions), checks_matched) # Filter checks further if options.gpu_only and options.cpu_only: printer.error("options `--gpu-only' and `--cpu-only' " "are mutually exclusive") sys.exit(1) if options.gpu_only: checks_matched = filter(filters.have_gpu_only(), checks_matched) elif options.cpu_only: checks_matched = filter(filters.have_cpu_only(), checks_matched) # Determine the allowed programming environments allowed_environs = { e.name for env_patt in options.prgenv for p in rt.system.partitions for e in p.environs if re.match(env_patt, e.name) } # Generate the test cases, validate dependencies and sort them checks_matched = list(checks_matched) testcases = generate_testcases(checks_matched, options.skip_system_check, options.skip_prgenv_check, allowed_environs) testgraph = dependency.build_deps(testcases) dependency.validate_deps(testgraph) testcases = dependency.toposort(testgraph) # Unload regression's module and load user-specified modules if hasattr(settings, 'reframe_module'): printer.warning( "the 'reframe_module' configuration option will be ignored; " "please use the '-u' or '--unload-module' options") if options.purge_env: rt.modules_system.unload_all() else: for m in options.unload_modules: rt.modules_system.unload_module(m) # Load the environment for the current system try: env.load(rt.system.preload_environ) except EnvironError as e: printer.error("failed to load current system's environment; " "please check your configuration") printer.debug(str(e)) raise for m in options.user_modules: try: rt.modules_system.load_module(m, force=True) except EnvironError as e: printer.warning("could not load module '%s' correctly: " "Skipping..." % m) printer.debug(str(e)) # Act on checks success = True if options.list: # List matched checks list_checks(list(checks_matched), printer) elif options.list_detailed: # List matched checks with details list_checks(list(checks_matched), printer, detailed=True) elif options.run: # Setup the execution policy if options.exec_policy == 'serial': exec_policy = SerialExecutionPolicy() elif options.exec_policy == 'async': exec_policy = AsynchronousExecutionPolicy() else: # This should not happen, since choices are handled by # argparser printer.error("unknown execution policy `%s': Exiting...") sys.exit(1) exec_policy.skip_system_check = options.skip_system_check exec_policy.force_local = options.force_local exec_policy.strict_check = options.strict exec_policy.skip_sanity_check = options.skip_sanity_check exec_policy.skip_performance_check = options.skip_performance_check exec_policy.keep_stage_files = options.keep_stage_files try: errmsg = "invalid option for --flex-alloc-tasks: '{0}'" sched_flex_alloc_tasks = int(options.flex_alloc_tasks) if sched_flex_alloc_tasks <= 0: raise ConfigError(errmsg.format(options.flex_alloc_tasks)) except ValueError: if not options.flex_alloc_tasks.lower() in {'idle', 'all'}: raise ConfigError(errmsg.format( options.flex_alloc_tasks)) from None sched_flex_alloc_tasks = options.flex_alloc_tasks exec_policy.sched_flex_alloc_tasks = sched_flex_alloc_tasks exec_policy.flex_alloc_tasks = options.flex_alloc_tasks exec_policy.sched_account = options.account exec_policy.sched_partition = options.partition exec_policy.sched_reservation = options.reservation exec_policy.sched_nodelist = options.nodelist exec_policy.sched_exclude_nodelist = options.exclude_nodes exec_policy.sched_options = options.job_options try: max_retries = int(options.max_retries) except ValueError: raise ConfigError('--max-retries is not a valid integer: %s' % max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: runner.runall(testcases) finally: # Print a retry report if we did any retries if runner.stats.failures(run=0): printer.info(runner.stats.retry_report()) # Print a failure report if we had failures in the last run if runner.stats.failures(): printer.info(runner.stats.failure_report()) success = False if options.performance_report: printer.info(runner.stats.performance_report()) else: printer.info('No action specified. Exiting...') printer.info("Try `%s -h' for a list of available actions." % argparser.prog) sys.exit(1) if not success: sys.exit(1) sys.exit(0) except KeyboardInterrupt: sys.exit(1) except ReframeError as e: printer.error(str(e)) sys.exit(1) except (Exception, ReframeFatalError): printer.error(format_exception(*sys.exc_info())) sys.exit(1) finally: try: if options.save_log_files: logging.save_log_files(rt.resources.output_prefix) except OSError as e: printer.error('could not save log file: %s' % e) sys.exit(1)