def ctx_sys(fixture_exec_ctx): yield rt.runtime().system
def tearDown(self): os_ext.rmtree(rt.runtime().resources.prefix) os_ext.rmtree('.rfm_testing', ignore_errors=True)
def main(): # Setup command line options argparser = argparse.ArgumentParser() output_options = argparser.add_argument_group( 'Options controlling ReFrame output') locate_options = argparser.add_argument_group( 'Options for discovering checks') select_options = argparser.add_argument_group( 'Options for selecting checks') action_options = argparser.add_argument_group( 'Options controlling actions') run_options = argparser.add_argument_group( 'Options controlling the execution of checks') env_options = argparser.add_argument_group( 'Options controlling the ReFrame environment') misc_options = argparser.add_argument_group('Miscellaneous options') # Output directory options output_options.add_argument('--prefix', action='store', metavar='DIR', help='Set general directory prefix to DIR', envvar='RFM_PREFIX', configvar='systems/prefix') output_options.add_argument('-o', '--output', action='store', metavar='DIR', help='Set output directory prefix to DIR', envvar='RFM_OUTPUT_DIR', configvar='systems/outputdir') output_options.add_argument('-s', '--stage', action='store', metavar='DIR', help='Set stage directory prefix to DIR', envvar='RFM_STAGE_DIR', configvar='systems/stagedir') output_options.add_argument( '--timestamp', action='store', nargs='?', const='', metavar='TIMEFMT', help=('Append a timestamp to the output and stage directory prefixes ' '(default: "%%FT%%T")'), envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs') output_options.add_argument( '--perflogdir', action='store', metavar='DIR', help=('Set performance log data directory prefix ' '(relevant only to the filelog log handler)'), envvar='RFM_PERFLOG_DIR', configvar='logging/handlers_perflog/filelog_basedir') output_options.add_argument( '--keep-stage-files', action='store_true', help='Keep stage directories even for successful checks', envvar='RFM_KEEP_STAGE_FILES', configvar='general/keep_stage_files') output_options.add_argument('--dont-restage', action='store_false', dest='clean_stagedir', help='Reuse the test stage directory', envvar='RFM_CLEAN_STAGEDIR', configvar='general/clean_stagedir') output_options.add_argument( '--save-log-files', action='store_true', default=False, help='Save ReFrame log files to the output directory', envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files') output_options.add_argument('--report-file', action='store', metavar='FILE', help="Store JSON run report in FILE", envvar='RFM_REPORT_FILE', configvar='general/report_file') output_options.add_argument('--report-junit', action='store', metavar='FILE', help="Store a JUnit report in FILE", envvar='RFM_REPORT_JUNIT', configvar='general/report_junit') # Check discovery options locate_options.add_argument('-c', '--checkpath', action='append', metavar='PATH', help="Add PATH to the check search path list", envvar='RFM_CHECK_SEARCH_PATH :', configvar='general/check_search_path') locate_options.add_argument( '-R', '--recursive', action='store_true', help='Search for checks in the search path recursively', envvar='RFM_CHECK_SEARCH_RECURSIVE', configvar='general/check_search_recursive') locate_options.add_argument( '--ignore-check-conflicts', action='store_true', help=('Skip checks with conflicting names ' '(this option is deprecated and has no effect)'), envvar='RFM_IGNORE_CHECK_CONFLICTS', configvar='general/ignore_check_conflicts') # Select options select_options.add_argument( '-t', '--tag', action='append', dest='tags', metavar='PATTERN', default=[], help='Select checks with at least one tag matching PATTERN') select_options.add_argument( '-n', '--name', action='append', dest='names', default=[], metavar='PATTERN', help='Select checks whose name matches PATTERN') select_options.add_argument( '-x', '--exclude', action='append', dest='exclude_names', metavar='PATTERN', default=[], help='Exclude checks whose name matches PATTERN') select_options.add_argument( '-p', '--prgenv', action='append', default=[r'.*'], metavar='PATTERN', help=('Select checks with at least one ' 'programming environment matching PATTERN')) select_options.add_argument( '--failed', action='store_true', help="Select failed test cases (only when '--restore-session' is used)" ) select_options.add_argument('--gpu-only', action='store_true', help='Select only GPU checks') select_options.add_argument('--cpu-only', action='store_true', help='Select only CPU checks') # Action options action_options.add_argument('-l', '--list', action='store_true', help='List the selected checks') action_options.add_argument( '-L', '--list-detailed', action='store_true', help='List the selected checks providing details for each test') action_options.add_argument( '--list-tags', action='store_true', help='List the unique tags found in the selected tests and exit') action_options.add_argument('-r', '--run', action='store_true', help='Run the selected checks') action_options.add_argument( '--ci-generate', action='store', metavar='FILE', help=('Generate into FILE a Gitlab CI pipeline ' 'for the selected tests and exit'), ) # Run options run_options.add_argument('-J', '--job-option', action='append', metavar='OPT', dest='job_options', default=[], help='Pass option OPT to job scheduler') run_options.add_argument('--force-local', action='store_true', help='Force local execution of checks') run_options.add_argument('--skip-sanity-check', action='store_true', help='Skip sanity checking') run_options.add_argument('--skip-performance-check', action='store_true', help='Skip performance checking') run_options.add_argument('--strict', action='store_true', help='Enforce strict performance checking') run_options.add_argument('--skip-system-check', action='store_true', help='Skip system check') run_options.add_argument('--skip-prgenv-check', action='store_true', help='Skip programming environment check') run_options.add_argument('-S', '--setvar', action='append', metavar='[TEST.]VAR=VAL', dest='vars', default=[], help=('Set test variable VAR to VAL in all tests ' 'or optionally in TEST only')) run_options.add_argument( '--exec-policy', metavar='POLICY', action='store', choices=['async', 'serial'], default='async', help='Set the execution policy of ReFrame (default: "async")') run_options.add_argument('--mode', action='store', help='Execution mode to use') run_options.add_argument( '--max-retries', metavar='NUM', action='store', default=0, help='Set the maximum number of times a failed regression test ' 'may be retried (default: 0)') run_options.add_argument('--maxfail', metavar='NUM', action='store', default=sys.maxsize, help='Exit after first NUM failures') run_options.add_argument('--restore-session', action='store', nargs='?', const='', metavar='REPORT', help='Restore a testing session from REPORT file') run_options.add_argument( '--flex-alloc-nodes', action='store', dest='flex_alloc_nodes', metavar='{all|STATE|NUM}', default=None, help='Set strategy for the flexible node allocation (default: "idle").' ) run_options.add_argument('--disable-hook', action='append', metavar='NAME', dest='hooks', default=[], help='Disable a pipeline hook for this run') # Environment options env_options.add_argument('-M', '--map-module', action='append', metavar='MAPPING', dest='module_mappings', default=[], help='Add a module mapping', envvar='RFM_MODULE_MAPPINGS ,', configvar='general/module_mappings') env_options.add_argument( '-m', '--module', action='append', default=[], metavar='MOD', dest='user_modules', help='Load module MOD before running any regression check', envvar='RFM_USER_MODULES ,', configvar='general/user_modules') env_options.add_argument('--module-mappings', action='store', metavar='FILE', dest='module_map_file', help='Load module mappings from FILE', envvar='RFM_MODULE_MAP_FILE', configvar='general/module_map_file') env_options.add_argument( '-u', '--unload-module', action='append', metavar='MOD', dest='unload_modules', default=[], help='Unload module MOD before running any regression check', envvar='RFM_UNLOAD_MODULES ,', configvar='general/unload_modules') env_options.add_argument( '--module-path', action='append', metavar='PATH', dest='module_paths', default=[], help='(Un)use module path PATH before running any regression check', ) env_options.add_argument( '--purge-env', action='store_true', dest='purge_env', default=False, help='Unload all modules before running any regression check', envvar='RFM_PURGE_ENVIRONMENT', configvar='general/purge_environment') env_options.add_argument( '--non-default-craype', action='store_true', help='Test a non-default Cray Programming Environment', envvar='RFM_NON_DEFAULT_CRAYPE', configvar='general/non_default_craype') # Miscellaneous options misc_options.add_argument('-C', '--config-file', action='store', dest='config_file', metavar='FILE', help='Set configuration file', envvar='RFM_CONFIG_FILE') misc_options.add_argument('--nocolor', action='store_false', dest='colorize', help='Disable coloring of output', envvar='RFM_COLORIZE', configvar='general/colorize') misc_options.add_argument('--failure-stats', action='store_true', help='Print failure statistics') misc_options.add_argument('--performance-report', action='store_true', help='Print a report for performance tests') misc_options.add_argument( '--show-config', action='store', nargs='?', const='all', metavar='PARAM', help='Print the value of configuration parameter PARAM and exit') misc_options.add_argument('--system', action='store', help='Load configuration for SYSTEM', envvar='RFM_SYSTEM') misc_options.add_argument('--detect-host-topology', action='store', nargs='?', const='-', help='Detect the local host topology and exit') misc_options.add_argument( '--upgrade-config-file', action='store', metavar='OLD[:NEW]', help='Upgrade ReFrame 2.x configuration file to ReFrame 3.x syntax') misc_options.add_argument('-V', '--version', action='version', version=osext.reframe_version()) misc_options.add_argument('-v', '--verbose', action='count', help='Increase verbosity level of output', envvar='RFM_VERBOSE', configvar='general/verbose') # Options not associated with command-line arguments argparser.add_argument( dest='graylog_server', envvar='RFM_GRAYLOG_ADDRESS', configvar='logging/handlers_perflog/graylog_address', help='Graylog server address') argparser.add_argument(dest='syslog_address', envvar='RFM_SYSLOG_ADDRESS', configvar='logging/handlers_perflog/syslog_address', help='Syslog server address') argparser.add_argument(dest='ignore_reqnodenotavail', envvar='RFM_IGNORE_REQNODENOTAVAIL', configvar='schedulers/ignore_reqnodenotavail', action='store_true', help='Graylog server address') argparser.add_argument(dest='use_login_shell', envvar='RFM_USE_LOGIN_SHELL', configvar='general/use_login_shell', action='store_true', help='Use a login shell for job scripts') argparser.add_argument(dest='resolve_module_conflicts', envvar='RFM_RESOLVE_MODULE_CONFLICTS', configvar='general/resolve_module_conflicts', action='store_true', help='Resolve module conflicts automatically') argparser.add_argument(dest='httpjson_url', envvar='RFM_HTTPJSON_URL', configvar='logging/handlers_perflog/httpjson_url', help='URL of HTTP server accepting JSON logs') argparser.add_argument(dest='remote_detect', envvar='RFM_REMOTE_DETECT', configvar='general/remote_detect', action='store_true', help='Detect remote system topology') argparser.add_argument( dest='remote_workdir', envvar='RFM_REMOTE_WORKDIR', configvar='general/remote_workdir', action='store', help='Working directory for launching ReFrame remotely') # Parse command line options = argparser.parse_args() if len(sys.argv) == 1: argparser.print_help() sys.exit(1) # First configure logging with our generic configuration so as to be able # to print pretty messages; logging will be reconfigured by user's # configuration later site_config = config.load_config( os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py')) site_config.select_subconfig('generic') options.update_config(site_config) logging.configure_logging(site_config) logging.getlogger().colorize = site_config.get('general/0/colorize') printer = PrettyPrinter() printer.colorize = site_config.get('general/0/colorize') printer.inc_verbosity(site_config.get('general/0/verbose')) if os.getenv('RFM_GRAYLOG_SERVER'): printer.warning( 'RFM_GRAYLOG_SERVER environment variable is deprecated; ' 'please use RFM_GRAYLOG_ADDRESS instead') os.environ['RFM_GRAYLOG_ADDRESS'] = os.getenv('RFM_GRAYLOG_SERVER') if options.upgrade_config_file is not None: old_config, *new_config = options.upgrade_config_file.split(':', maxsplit=1) new_config = new_config[0] if new_config else None try: new_config = config.convert_old_config(old_config, new_config) except Exception as e: printer.error(f'could not convert file: {e}') sys.exit(1) printer.info(f'Conversion successful! ' f'The converted file can be found at {new_config!r}.') sys.exit(0) # Now configure ReFrame according to the user configuration file try: try: printer.debug('Loading user configuration') site_config = config.load_config(options.config_file) except warnings.ReframeDeprecationWarning as e: printer.warning(e) converted = config.convert_old_config(options.config_file) printer.warning(f"configuration file has been converted " f"to the new syntax here: '{converted}'") site_config = config.load_config(converted) site_config.validate() # We ignore errors about unresolved sections or configuration # parameters here, because they might be defined at the individual # partition level and will be caught when we will instantiating # internally the system and partitions later on. site_config.select_subconfig(options.system, ignore_resolve_errors=True) for err in options.update_config(site_config): printer.warning(str(err)) # Update options from the selected execution mode if options.mode: mode_args = site_config.get(f'modes/@{options.mode}/options') # We lexically split the mode options, because otherwise spaces # will be treated as part of the option argument; see GH bug #1554 mode_args = list( itertools.chain.from_iterable( shlex.split(m) for m in mode_args)) # Parse the mode's options and reparse the command-line options = argparser.parse_args(mode_args) options = argparser.parse_args(namespace=options.cmd_options) options.update_config(site_config) logging.configure_logging(site_config) except (OSError, errors.ConfigError) as e: printer.error(f'failed to load configuration: {e}') printer.error(logfiles_message()) sys.exit(1) logging.getlogger().colorize = site_config.get('general/0/colorize') printer.colorize = site_config.get('general/0/colorize') printer.inc_verbosity(site_config.get('general/0/verbose')) try: printer.debug('Initializing runtime') runtime.init_runtime(site_config) except errors.ConfigError as e: printer.error(f'failed to initialize runtime: {e}') printer.error(logfiles_message()) sys.exit(1) if site_config.get('general/0/ignore_check_conflicts'): logging.getlogger().warning( "the 'ignore_check_conflicts' option is deprecated " "and will be removed in the future") rt = runtime.runtime() autodetect.detect_topology() try: if site_config.get('general/0/module_map_file'): rt.modules_system.load_mapping_from_file( site_config.get('general/0/module_map_file')) if site_config.get('general/0/module_mappings'): for m in site_config.get('general/0/module_mappings'): rt.modules_system.load_mapping(m) except (errors.ConfigError, OSError) as e: printer.error('could not load module mappings: %s' % e) sys.exit(1) if (osext.samefile(rt.stage_prefix, rt.output_prefix) and not site_config.get('general/0/keep_stage_files')): printer.error("stage and output refer to the same directory; " "if this is on purpose, please use the " "'--keep-stage-files' option.") printer.error(logfiles_message()) sys.exit(1) # Show configuration after everything is set up if options.show_config: config_param = options.show_config if config_param == 'all': printer.info(str(rt.site_config)) else: value = rt.get_option(config_param) if value is None: printer.error( f'no such configuration parameter found: {config_param}') else: printer.info(json.dumps(value, indent=2)) sys.exit(0) if options.detect_host_topology: from reframe.utility.cpuinfo import cpuinfo topofile = options.detect_host_topology if topofile == '-': json.dump(cpuinfo(), sys.stdout, indent=2) sys.stdout.write('\n') else: try: with open(topofile, 'w') as fp: json.dump(cpuinfo(), fp, indent=2) fp.write('\n') except OSError as e: getlogger().error( f'could not write topology file: {topofile!r}') sys.exit(1) sys.exit(0) printer.debug(format_env(options.env_vars)) # Setup the check loader if options.restore_session is not None: # We need to load the failed checks only from a list of reports if options.restore_session: filenames = options.restore_session.split(',') else: filenames = [ runreport.next_report_filename(osext.expandvars( site_config.get('general/0/report_file')), new=False) ] report = runreport.load_report(*filenames) check_search_path = list(report.slice('filename', unique=True)) check_search_recursive = False # If `-c` or `-R` are passed explicitly outside the configuration # file, override the values set from the report file if site_config.is_sticky_option('general/check_search_path'): printer.warning( 'Ignoring check search path set in the report file: ' 'search path set explicitly in the command-line or ' 'the environment') check_search_path = site_config.get('general/0/check_search_path') if site_config.is_sticky_option('general/check_search_recursive'): printer.warning( 'Ignoring check search recursive option from the report file: ' 'option set explicitly in the command-line or the environment') check_search_recursive = site_config.get( 'general/0/check_search_recursive') else: check_search_recursive = site_config.get( 'general/0/check_search_recursive') check_search_path = site_config.get('general/0/check_search_path') # Collect any variables set from the command line external_vars = {} for expr in options.vars: try: lhs, rhs = expr.split('=', maxsplit=1) except ValueError: printer.warning( f'invalid test variable assignment: {expr!r}; skipping') else: external_vars[lhs] = rhs loader = RegressionCheckLoader(check_search_path, check_search_recursive, external_vars) def print_infoline(param, value): param = param + ':' printer.info(f" {param.ljust(18)} {value}") session_info = { 'cmdline': ' '.join(sys.argv), 'config_file': rt.site_config.filename, 'data_version': runreport.DATA_VERSION, 'hostname': socket.gethostname(), 'prefix_output': rt.output_prefix, 'prefix_stage': rt.stage_prefix, 'user': osext.osuser(), 'version': osext.reframe_version(), 'workdir': os.getcwd(), } # Print command line printer.info(f"[ReFrame Setup]") print_infoline('version', session_info['version']) print_infoline('command', repr(session_info['cmdline'])) print_infoline( f"launched by", f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}") print_infoline('working directory', repr(session_info['workdir'])) print_infoline('settings file', f"{session_info['config_file']!r}") print_infoline( 'check search path', f"{'(R) ' if loader.recurse else ''}" f"{':'.join(loader.load_path)!r}") print_infoline('stage directory', repr(session_info['prefix_stage'])) print_infoline('output directory', repr(session_info['prefix_output'])) printer.info('') try: # Locate and load checks checks_found = loader.load_all() printer.verbose(f'Loaded {len(checks_found)} test(s)') # Generate all possible test cases first; we will need them for # resolving dependencies after filtering # Determine the allowed programming environments allowed_environs = { e.name for env_patt in options.prgenv for p in rt.system.partitions for e in p.environs if re.match(env_patt, e.name) } testcases_all = generate_testcases(checks_found, options.skip_system_check, options.skip_prgenv_check, allowed_environs) testcases = testcases_all printer.verbose(f'Generated {len(testcases)} test case(s)') # Filter test cases by name if options.exclude_names: for name in options.exclude_names: testcases = filter(filters.have_not_name(name), testcases) if options.names: testcases = filter(filters.have_name('|'.join(options.names)), testcases) testcases = list(testcases) printer.verbose( f'Filtering test cases(s) by name: {len(testcases)} remaining') # Filter test cases by tags for tag in options.tags: testcases = filter(filters.have_tag(tag), testcases) testcases = list(testcases) printer.verbose( f'Filtering test cases(s) by tags: {len(testcases)} remaining') # Filter test cases further if options.gpu_only and options.cpu_only: printer.error("options `--gpu-only' and `--cpu-only' " "are mutually exclusive") sys.exit(1) if options.gpu_only: testcases = filter(filters.have_gpu_only(), testcases) elif options.cpu_only: testcases = filter(filters.have_cpu_only(), testcases) testcases = list(testcases) printer.verbose(f'Filtering test cases(s) by other attributes: ' f'{len(testcases)} remaining') # Filter in failed cases if options.failed: if options.restore_session is None: printer.error( "the option '--failed' can only be used " "in combination with the '--restore-session' option") sys.exit(1) def _case_failed(t): rec = report.case(*t) if not rec: return False return (rec['result'] == 'failure' or rec['result'] == 'aborted') testcases = list(filter(_case_failed, testcases)) printer.verbose(f'Filtering successful test case(s): ' f'{len(testcases)} remaining') # Prepare for running printer.debug('Building and validating the full test DAG') testgraph, skipped_cases = dependencies.build_deps(testcases_all) if skipped_cases: # Some cases were skipped, so adjust testcases testcases = list(set(testcases) - set(skipped_cases)) printer.verbose( f'Filtering test case(s) due to unresolved dependencies: ' f'{len(testcases)} remaining') dependencies.validate_deps(testgraph) printer.debug('Full test DAG:') printer.debug(dependencies.format_deps(testgraph)) restored_cases = [] if len(testcases) != len(testcases_all): testgraph = dependencies.prune_deps( testgraph, testcases, max_depth=1 if options.restore_session is not None else None) printer.debug('Pruned test DAG') printer.debug(dependencies.format_deps(testgraph)) if options.restore_session is not None: testgraph, restored_cases = report.restore_dangling(testgraph) testcases = dependencies.toposort(testgraph, is_subgraph=options.restore_session is not None) printer.verbose(f'Final number of test cases: {len(testcases)}') # Disable hooks for tc in testcases: for h in options.hooks: tc.check.disable_hook(h) # Act on checks if options.list or options.list_detailed: list_checks(testcases, printer, options.list_detailed) sys.exit(0) if options.list_tags: list_tags(testcases, printer) sys.exit(0) if options.ci_generate: list_checks(testcases, printer) printer.info('[Generate CI]') with open(options.ci_generate, 'wt') as fp: ci.emit_pipeline(fp, testcases) printer.info(f' Gitlab pipeline generated successfully ' f'in {options.ci_generate!r}.\n') sys.exit(0) if not options.run: printer.error("No action option specified. Available options:\n" " - `-l'/`-L' for listing\n" " - `-r' for running\n" " - `--list-tags' for listing unique test tags\n" " - `--ci-generate' for generating a CI pipeline\n" f"Try `{argparser.prog} -h' for more options.") sys.exit(1) # Manipulate ReFrame's environment if site_config.get('general/0/purge_environment'): rt.modules_system.unload_all() else: for m in site_config.get('general/0/unload_modules'): rt.modules_system.unload_module(**m) # Load the environment for the current system try: printer.debug(f'Loading environment for current system') runtime.loadenv(rt.system.preload_environ) except errors.EnvironError as e: printer.error("failed to load current system's environment; " "please check your configuration") printer.debug(str(e)) raise def module_use(*paths): try: rt.modules_system.searchpath_add(*paths) except errors.EnvironError as e: printer.warning(f'could not add module paths correctly') printer.debug(str(e)) def module_unuse(*paths): try: rt.modules_system.searchpath_remove(*paths) except errors.EnvironError as e: printer.warning(f'could not remove module paths correctly') printer.debug(str(e)) printer.debug('(Un)using module paths from command line') module_paths = {} for d in options.module_paths: if d.startswith('-'): module_paths.setdefault('-', []) module_paths['-'].append(d[1:]) elif d.startswith('+'): module_paths.setdefault('+', []) module_paths['+'].append(d[1:]) else: module_paths.setdefault('x', []) module_paths['x'].append(d) for op, paths in module_paths.items(): if op == '+': module_use(*paths) elif op == '-': module_unuse(*paths) else: # First empty the current module path in a portable way searchpath = [p for p in rt.modules_system.searchpath if p] if searchpath: rt.modules_system.searchpath_remove(*searchpath) # Treat `A:B` syntax as well in this case paths = itertools.chain(*(p.split(':') for p in paths)) module_use(*paths) printer.debug('Loading user modules from command line') for m in site_config.get('general/0/user_modules'): try: rt.modules_system.load_module(**m, force=True) except errors.EnvironError as e: printer.warning( f'could not load module {m["name"]!r} correctly; ' f'skipping...') printer.debug(str(e)) options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle' # Run the tests # Setup the execution policy if options.exec_policy == 'serial': exec_policy = SerialExecutionPolicy() elif options.exec_policy == 'async': exec_policy = AsynchronousExecutionPolicy() else: # This should not happen, since choices are handled by # argparser printer.error("unknown execution policy `%s': Exiting...") sys.exit(1) exec_policy.skip_system_check = options.skip_system_check exec_policy.force_local = options.force_local exec_policy.strict_check = options.strict exec_policy.skip_sanity_check = options.skip_sanity_check exec_policy.skip_performance_check = options.skip_performance_check exec_policy.keep_stage_files = site_config.get( 'general/0/keep_stage_files') try: errmsg = "invalid option for --flex-alloc-nodes: '{0}'" sched_flex_alloc_nodes = int(options.flex_alloc_nodes) if sched_flex_alloc_nodes <= 0: raise errors.ConfigError( errmsg.format(options.flex_alloc_nodes)) except ValueError: sched_flex_alloc_nodes = options.flex_alloc_nodes exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes parsed_job_options = [] for opt in options.job_options: opt_split = opt.split('=', maxsplit=1) optstr = opt_split[0] valstr = opt_split[1] if len(opt_split) > 1 else '' if opt.startswith('-') or opt.startswith('#'): parsed_job_options.append(opt) elif len(optstr) == 1: parsed_job_options.append(f'-{optstr} {valstr}') else: parsed_job_options.append(f'--{optstr} {valstr}') exec_policy.sched_options = parsed_job_options try: max_retries = int(options.max_retries) except ValueError: raise errors.ConfigError( f'--max-retries is not a valid integer: {max_retries}' ) from None try: max_failures = int(options.maxfail) if max_failures < 0: raise errors.ConfigError( f'--maxfail should be a non-negative integer: ' f'{options.maxfail!r}') except ValueError: raise errors.ConfigError( f'--maxfail is not a valid integer: {options.maxfail!r}' ) from None runner = Runner(exec_policy, printer, max_retries, max_failures) try: time_start = time.time() session_info['time_start'] = time.strftime( '%FT%T%z', time.localtime(time_start), ) runner.runall(testcases, restored_cases) finally: time_end = time.time() session_info['time_end'] = time.strftime('%FT%T%z', time.localtime(time_end)) session_info['time_elapsed'] = time_end - time_start # Print a retry report if we did any retries if runner.stats.failed(run=0): printer.info(runner.stats.retry_report()) # Print a failure report if we had failures in the last run success = True if runner.stats.failed(): success = False runner.stats.print_failure_report(printer) if options.failure_stats: runner.stats.print_failure_stats(printer) if options.performance_report: printer.info(runner.stats.performance_report()) # Generate the report for this session report_file = os.path.normpath( osext.expandvars(rt.get_option('general/0/report_file'))) basedir = os.path.dirname(report_file) if basedir: os.makedirs(basedir, exist_ok=True) # Build final JSON report run_stats = runner.stats.json() session_info.update({ 'num_cases': run_stats[0]['num_cases'], 'num_failures': run_stats[-1]['num_failures'] }) json_report = { 'session_info': session_info, 'runs': run_stats, 'restored_cases': [] } if options.restore_session is not None: for c in restored_cases: json_report['restored_cases'].append(report.case(*c)) report_file = runreport.next_report_filename(report_file) try: with open(report_file, 'w') as fp: jsonext.dump(json_report, fp, indent=2) fp.write('\n') printer.info(f'Run report saved in {report_file!r}') except OSError as e: printer.warning( f'failed to generate report in {report_file!r}: {e}') # Generate the junit xml report for this session junit_report_file = rt.get_option('general/0/report_junit') if junit_report_file: # Expand variables in filename junit_report_file = osext.expandvars(junit_report_file) junit_xml = runreport.junit_xml_report(json_report) try: with open(junit_report_file, 'w') as fp: runreport.junit_dump(junit_xml, fp) except OSError as e: printer.warning( f'failed to generate report in {junit_report_file!r}: ' f'{e}') if not success: sys.exit(1) sys.exit(0) except (Exception, KeyboardInterrupt, errors.ReframeFatalError): exc_info = sys.exc_info() tb = ''.join(traceback.format_exception(*exc_info)) printer.error(f'run session stopped: {errors.what(*exc_info)}') if errors.is_exit_request(*exc_info): # Print stack traces for exit requests only when TOO verbose printer.debug2(tb) elif errors.is_severe(*exc_info): printer.error(tb) else: printer.verbose(tb) sys.exit(1) finally: try: log_files = logging.log_files() if site_config.get('general/0/save_log_files'): log_files = logging.save_log_files(rt.output_prefix) except OSError as e: printer.error(f'could not save log file: {e}') sys.exit(1) finally: printer.info(logfiles_message())
def partition_by_name(name): for p in rt.runtime().system.partitions: if p.name == name: return p return None
def setup_local_execution(self): self.partition = rt.runtime().system.partition('login') self.progenv = self.partition.environment('builtin-gcc')
def temp_prefix(self): # Set runtime prefix rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def __init__(self): self._prefix = '#BSUB' self._submit_timeout = rt.runtime().get_option( f'schedulers/@{self.registered_name}/job_submit_timeout' )
def test_global_config(basic_config): rlog.configure_logging(rt.runtime().site_config) assert rlog.getlogger() is not rlog.null_logger
def check_sanity(self): return runtime().modules_system.is_module_loaded('PrgEnv-cray')
def test_handler_level(basic_config, logfile): rlog.configure_logging(rt.runtime().site_config) rlog.getlogger().info('foo') rlog.getlogger().warning('bar') assert not _found_in_logfile('foo', logfile) assert _found_in_logfile('bar', logfile)
def test_date_format(basic_config, logfile): rlog.configure_logging(rt.runtime().site_config) rlog.getlogger().warning('foo') assert _found_in_logfile(datetime.now().strftime('%F'), logfile)
def test_valid_level(basic_config): rlog.configure_logging(rt.runtime().site_config) assert rlog.INFO == rlog.getlogger().getEffectiveLevel()
def test_autotect_with_invalid_files(invalid_topo_exec_ctx): autodetect.detect_topology() part = runtime().system.partitions[0] assert part.processor.info == cpuinfo() assert part.devices == []
def test_partition(self): p = rt.runtime().system.partition('gpu') assert 2 == self.count_checks(filters.have_partition([p])) p = rt.runtime().system.partition('login') assert 0 == self.count_checks(filters.have_partition([p]))
def setUp(self): self.partition = rt.runtime().system.partition('login') self.prgenv = self.partition.environment('builtin-gcc') # Set runtime prefix rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def user_runtime(): if fixtures.USER_CONFIG_FILE: with rt.temp_runtime(fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM): yield rt.runtime() else: yield rt.runtime()
def tearDown(self): os_ext.rmtree(rt.runtime().resources.prefix)
def assert_modules_loaded(modules): modsys = rt.runtime().modules_system for m in modules: assert modsys.is_module_loaded(m)
def test_sarus(self): partition, environ = _setup_remote_execution() self._skip_if_not_configured(partition, 'Sarus') with tempfile.TemporaryDirectory(dir='unittests') as dirname: rt.runtime().resources.prefix = dirname _run(self.create_test('Sarus', 'ubuntu:18.04'), partition, environ)
def find_modules(substr, environ_mapping=None): '''Return all modules in the current system that contain ``substr`` in their name. This function is a generator and will yield tuples of partition, environment and module combinations for each partition of the current system and for each environment of a partition. The ``environ_mapping`` argument allows you to map module name patterns to ReFrame environments. This is useful for flat module name schemes, in order to avoid incompatible combinations of modules and environments. You can use this function to parametrize regression tests over the available environment modules. The following example will generate tests for all the available ``netcdf`` packages in the system: .. code:: python @rfm.simple_test class MyTest(rfm.RegressionTest): module_info = parameter(find_modules('netcdf')) @rfm.run_after('init') def apply_module_info(self): s, e, m = self.module_info self.valid_systems = [s] self.valid_prog_environs = [e] self.modules = [m] ... The following example shows the use of ``environ_mapping`` with flat module name schemes. In this example, the toolchain for which the package was built is encoded in the module's name. Using the ``environ_mapping`` argument we can map module name patterns to ReFrame environments, so that invalid combinations are pruned: .. code:: python my_find_modules = functools.partial(find_modules, environ_mapping={ r'.*CrayGNU.*': 'PrgEnv-gnu', r'.*CrayIntel.*': 'PrgEnv-intel', r'.*CrayCCE.*': 'PrgEnv-cray' }) @rfm.simple_test class MyTest(rfm.RegressionTest): module_info = parameter(my_find_modules('GROMACS')) @rfm.run_after('init') def apply_module_info(self): s, e, m = self.module_info self.valid_systems = [s] self.valid_prog_environs = [e] self.modules = [m] ... :arg substr: A substring that the returned module names must contain. :arg environ_mapping: A dictionary mapping regular expressions to environment names. :returns: An iterator that iterates over tuples of the module, partition and environment name combinations that were found. ''' import reframe.core.runtime as rt if not isinstance(substr, str): raise TypeError("'substr' argument must be a string") if (environ_mapping is not None and not isinstance(environ_mapping, typ.Dict[str, str])): raise TypeError( "'environ_mapping' argument must be of type Dict[str,str]" ) def _is_valid_for_env(m, e): if environ_mapping is None: return True for patt, env in environ_mapping.items(): if re.match(patt, m) and e == env: return True return False ms = rt.runtime().modules_system current_system = rt.runtime().system snap0 = rt.snapshot() for p in current_system.partitions: for e in p.environs: rt.loadenv(p.local_env, e) modules = ms.available_modules(substr) snap0.restore() for m in modules: if _is_valid_for_env(m, e.name): yield (p.fullname, e.name, m)
def add_task(self, task): current_run = rt.runtime().current_run if current_run == len(self._tasks): self._tasks.append([]) self._tasks[current_run].append(task)
def __init__(self, name='env_snapshot'): super().__init__(name, runtime().modules_system.loaded_modules(), os.environ.items())
def has_sane_modules_system(): return not isinstance(rt.runtime().modules_system.backend, (modules.NoModImpl, modules.SpackImpl))
def _temp_runtime(config_file, system=None, options={}): options.update({'systems/prefix': str(tmp_path)}) with rt.temp_runtime(config_file, system, options): yield rt.runtime()
def setUp(self): self.setup_local_execution() self.loader = RegressionCheckLoader(['unittests/resources/checks']) # Set runtime prefix rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def set_max_jobs(self, value): for p in rt.runtime().system.partitions: p._max_jobs = value
def main(): # Setup command line options argparser = argparse.ArgumentParser() output_options = argparser.add_argument_group('Options controlling output') locate_options = argparser.add_argument_group( 'Options for locating checks') select_options = argparser.add_argument_group( 'Options for selecting checks') action_options = argparser.add_argument_group( 'Options controlling actions') run_options = argparser.add_argument_group( 'Options controlling execution of checks') env_options = argparser.add_argument_group( 'Options controlling environment') misc_options = argparser.add_argument_group('Miscellaneous options') # Output directory options output_options.add_argument('--prefix', action='store', metavar='DIR', help='Set output directory prefix to DIR') output_options.add_argument('-o', '--output', action='store', metavar='DIR', help='Set output directory to DIR') output_options.add_argument('-s', '--stage', action='store', metavar='DIR', help='Set stage directory to DIR') output_options.add_argument( '--perflogdir', action='store', metavar='DIR', help='Set directory prefix for the performance logs ' '(default: ${prefix}/perflogs, ' 'relevant only if the filelog backend is used)') output_options.add_argument( '--keep-stage-files', action='store_true', help='Keep stage directory even if check is successful') output_options.add_argument( '--save-log-files', action='store_true', default=False, help='Copy the log file from the work dir to the output dir at the ' 'end of the program') # Check discovery options locate_options.add_argument( '-c', '--checkpath', action='store', metavar='DIR|FILE', help="Search for checks in DIR or FILE; multiple paths can be " "separated with `:'") locate_options.add_argument('-R', '--recursive', action='store_true', help='Load checks recursively') locate_options.add_argument('--ignore-check-conflicts', action='store_true', help='Skip checks with conflicting names') # Select options select_options.add_argument('-t', '--tag', action='append', dest='tags', default=[], help='Select checks matching TAG') select_options.add_argument('-n', '--name', action='append', dest='names', default=[], metavar='NAME', help='Select checks with NAME') select_options.add_argument('-x', '--exclude', action='append', dest='exclude_names', metavar='NAME', default=[], help='Exclude checks with NAME') select_options.add_argument( '-p', '--prgenv', action='append', default=[r'.*'], help='Select tests for PRGENV programming environment only') select_options.add_argument('--gpu-only', action='store_true', help='Select only GPU tests') select_options.add_argument('--cpu-only', action='store_true', help='Select only CPU tests') # Action options action_options.add_argument('-l', '--list', action='store_true', help='List matched regression checks') action_options.add_argument( '-L', '--list-detailed', action='store_true', help='List matched regression checks with a detailed description') action_options.add_argument('-r', '--run', action='store_true', help='Run regression with the selected checks') # Run options run_options.add_argument('-A', '--account', action='store', help='Use ACCOUNT for submitting jobs') run_options.add_argument('-P', '--partition', action='store', metavar='PART', help='Use PART for submitting jobs') run_options.add_argument('--reservation', action='store', metavar='RES', help='Use RES for submitting jobs') run_options.add_argument('--nodelist', action='store', help='Run checks on the selected list of nodes') run_options.add_argument( '--exclude-nodes', action='store', metavar='NODELIST', help='Exclude the list of nodes from running checks') run_options.add_argument('--job-option', action='append', metavar='OPT', dest='job_options', default=[], help='Pass OPT to job scheduler') run_options.add_argument('--force-local', action='store_true', help='Force local execution of checks') run_options.add_argument('--skip-sanity-check', action='store_true', help='Skip sanity checking') run_options.add_argument('--skip-performance-check', action='store_true', help='Skip performance checking') run_options.add_argument('--strict', action='store_true', help='Force strict performance checking') run_options.add_argument('--skip-system-check', action='store_true', help='Skip system check') run_options.add_argument('--skip-prgenv-check', action='store_true', help='Skip prog. environment check') run_options.add_argument( '--exec-policy', metavar='POLICY', action='store', choices=['async', 'serial'], default='async', help='Specify the execution policy for running the regression tests. ' 'Available policies: "async" (default), "serial"') run_options.add_argument('--mode', action='store', help='Execution mode to use') run_options.add_argument( '--max-retries', metavar='NUM', action='store', default=0, help='Specify the maximum number of times a failed regression test ' 'may be retried (default: 0)') run_options.add_argument( '--flex-alloc-tasks', action='store', dest='flex_alloc_tasks', metavar='{all|idle|NUM}', default=None, help='*deprecated*, please use --flex-alloc-nodes instead') run_options.add_argument( '--flex-alloc-nodes', action='store', dest='flex_alloc_nodes', metavar='{all|idle|NUM}', default=None, help="Strategy for flexible node allocation (default: 'idle').") env_options.add_argument('-M', '--map-module', action='append', metavar='MAPPING', dest='module_mappings', default=[], help='Apply a single module mapping') env_options.add_argument( '-m', '--module', action='append', default=[], metavar='MOD', dest='user_modules', help='Load module MOD before running the regression suite') env_options.add_argument('--module-mappings', action='store', metavar='FILE', dest='module_map_file', help='Apply module mappings defined in FILE') env_options.add_argument( '-u', '--unload-module', action='append', metavar='MOD', dest='unload_modules', default=[], help='Unload module MOD before running the regression suite') env_options.add_argument( '--purge-env', action='store_true', dest='purge_env', default=False, help='Purge environment before running the regression suite') # Miscellaneous options misc_options.add_argument( '-C', '--config-file', action='store', dest='config_file', metavar='FILE', default=os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py'), help='Specify a custom config-file for the machine. ' '(default: %s' % os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py')) misc_options.add_argument('--nocolor', action='store_false', dest='colorize', default=True, help='Disable coloring of output') misc_options.add_argument('--performance-report', action='store_true', help='Print the performance report') # FIXME: This should move to env_options as soon as # https://github.com/eth-cscs/reframe/pull/946 is merged misc_options.add_argument('--non-default-craype', action='store_true', default=False, help='Test a non-default Cray PE') misc_options.add_argument( '--show-config', action='store_true', help='Print configuration of the current system and exit') misc_options.add_argument( '--show-config-env', action='store', metavar='ENV', help='Print configuration of environment ENV and exit') misc_options.add_argument('--system', action='store', help='Load SYSTEM configuration explicitly') misc_options.add_argument( '--timestamp', action='store', nargs='?', const='%FT%T', metavar='TIMEFMT', help='Append a timestamp component to the regression directories' '(default format "%%FT%%T")') misc_options.add_argument('-V', '--version', action='version', version=os_ext.reframe_version()) misc_options.add_argument('-v', '--verbose', action='count', default=0, help='Increase verbosity level of output') if len(sys.argv) == 1: argparser.print_help() sys.exit(1) # Parse command line options = argparser.parse_args() # Load configuration try: settings = config.load_settings_from_file(options.config_file) except (OSError, ReframeError) as e: sys.stderr.write('%s: could not load settings: %s\n' % (sys.argv[0], e)) sys.exit(1) # Configure logging try: logging.configure_logging(settings.logging_config), except (OSError, ConfigError) as e: sys.stderr.write('could not configure logging: %s\n' % e) sys.exit(1) # Set colors in logger logging.getlogger().colorize = options.colorize # Setup printer printer = PrettyPrinter() printer.colorize = options.colorize if options.verbose: printer.inc_verbosity(options.verbose) try: runtime.init_runtime(settings.site_configuration, options.system, non_default_craype=options.non_default_craype) except SystemAutodetectionError: printer.warning( 'could not find a configuration entry for the current system; ' 'falling back to a generic system configuration; ' 'please check the online documentation on how to configure ' 'ReFrame for your system.') settings.site_configuration['systems'] = { 'generic': { 'descr': 'Generic fallback system configuration', 'hostnames': ['localhost'], 'partitions': { 'login': { 'scheduler': 'local', 'environs': ['builtin-gcc'], 'descr': 'Login nodes' } } } } settings.site_configuration['environments'] = { '*': { 'builtin-gcc': { 'type': 'ProgEnvironment', 'cc': 'gcc', 'cxx': 'g++', 'ftn': 'gfortran', } } } runtime.init_runtime(settings.site_configuration, 'generic', non_default_craype=options.non_default_craype) except Exception as e: printer.error('configuration error: %s' % e) printer.verbose(''.join(traceback.format_exception(*sys.exc_info()))) sys.exit(1) rt = runtime.runtime() try: if options.module_map_file: rt.modules_system.load_mapping_from_file(options.module_map_file) if options.module_mappings: for m in options.module_mappings: rt.modules_system.load_mapping(m) except (ConfigError, OSError) as e: printer.error('could not load module mappings: %s' % e) sys.exit(1) if options.mode: try: mode_args = rt.mode(options.mode) # Parse the mode's options and reparse the command-line options = argparser.parse_args(mode_args) options = argparser.parse_args(namespace=options) except ConfigError as e: printer.error('could not obtain execution mode: %s' % e) sys.exit(1) # Adjust system directories if options.prefix: # if prefix is set, reset all other directories rt.resources.prefix = os_ext.expandvars(options.prefix) rt.resources.outputdir = None rt.resources.stagedir = None if options.output: rt.resources.outputdir = os_ext.expandvars(options.output) if options.stage: rt.resources.stagedir = os_ext.expandvars(options.stage) if (os_ext.samefile(rt.resources.stage_prefix, rt.resources.output_prefix) and not options.keep_stage_files): printer.error('stage and output refer to the same directory; ' 'if this is on purpose, please use also the ' "`--keep-stage-files' option.") sys.exit(1) if options.timestamp: rt.resources.timefmt = options.timestamp # Configure performance logging # NOTE: we need resources to be configured in order to set the global # perf. logging prefix correctly if options.perflogdir: rt.resources.perflogdir = os_ext.expandvars(options.perflogdir) logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = ( rt.resources.perflog_prefix) # Show configuration after everything is set up if options.show_config: printer.info(rt.show_config()) sys.exit(0) if options.show_config_env: envname = options.show_config_env for p in rt.system.partitions: environ = p.environment(envname) if environ: break if environ is None: printer.error('no such environment: ' + envname) sys.exit(1) printer.info(environ.details()) sys.exit(0) if hasattr(settings, 'perf_logging_config'): try: logging.configure_perflogging(settings.perf_logging_config) except (OSError, ConfigError) as e: printer.error('could not configure performance logging: %s\n' % e) sys.exit(1) else: printer.warning('no performance logging is configured; ' 'please check documentation') # Setup the check loader if options.checkpath: load_path = [] for d in options.checkpath.split(':'): d = os_ext.expandvars(d) if not os.path.exists(d): printer.warning("%s: path `%s' does not exist. Skipping..." % (argparser.prog, d)) continue load_path.append(os.path.realpath(d)) load_path = os_ext.unique_abs_paths(load_path, prune_children=options.recursive) loader = RegressionCheckLoader( load_path, recurse=options.recursive, ignore_conflicts=options.ignore_check_conflicts) else: loader = RegressionCheckLoader(load_path=settings.checks_path, prefix=reframe.INSTALL_PREFIX, recurse=settings.checks_path_recurse) printer.debug(argparse.format_options(options)) # Print command line printer.info('Command line: %s' % ' '.join(sys.argv)) printer.info('Reframe version: ' + reframe.VERSION) printer.info('Launched by user: '******'<unknown>')) printer.info('Launched on host: ' + socket.gethostname()) # Print important paths printer.info('Reframe paths') printer.info('=============') printer.info(' Check prefix : %s' % loader.prefix) printer.info( '%03s Check search path : %s' % ('(R)' if loader.recurse else '', "'%s'" % ':'.join(loader.load_path))) printer.info(' Stage dir prefix : %s' % rt.resources.stage_prefix) printer.info(' Output dir prefix : %s' % rt.resources.output_prefix) printer.info( ' Perf. logging prefix : %s' % os.path.abspath(logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'])) try: # Locate and load checks try: checks_found = loader.load_all() except OSError as e: raise ReframeError from e # Filter checks by name checks_matched = checks_found if options.exclude_names: for name in options.exclude_names: checks_matched = filter(filters.have_not_name(name), checks_matched) if options.names: checks_matched = filter(filters.have_name('|'.join(options.names)), checks_matched) # Filter checks by tags for tag in options.tags: checks_matched = filter(filters.have_tag(tag), checks_matched) # Filter checks by prgenv if not options.skip_prgenv_check: for prgenv in options.prgenv: checks_matched = filter(filters.have_prgenv(prgenv), checks_matched) # Filter checks by system if not options.skip_system_check: checks_matched = filter( filters.have_partition(rt.system.partitions), checks_matched) # Filter checks further if options.gpu_only and options.cpu_only: printer.error("options `--gpu-only' and `--cpu-only' " "are mutually exclusive") sys.exit(1) if options.gpu_only: checks_matched = filter(filters.have_gpu_only(), checks_matched) elif options.cpu_only: checks_matched = filter(filters.have_cpu_only(), checks_matched) # Determine the allowed programming environments allowed_environs = { e.name for env_patt in options.prgenv for p in rt.system.partitions for e in p.environs if re.match(env_patt, e.name) } # Generate the test cases, validate dependencies and sort them checks_matched = list(checks_matched) testcases = generate_testcases(checks_matched, options.skip_system_check, options.skip_prgenv_check, allowed_environs) testgraph = dependency.build_deps(testcases) dependency.validate_deps(testgraph) testcases = dependency.toposort(testgraph) # Unload regression's module and load user-specified modules if hasattr(settings, 'reframe_module'): printer.warning( "the 'reframe_module' configuration option will be ignored; " "please use the '-u' or '--unload-module' options") if options.purge_env: rt.modules_system.unload_all() else: for m in options.unload_modules: rt.modules_system.unload_module(m) # Load the environment for the current system try: env.load(rt.system.preload_environ) except EnvironError as e: printer.error("failed to load current system's environment; " "please check your configuration") printer.debug(str(e)) raise for m in options.user_modules: try: rt.modules_system.load_module(m, force=True) except EnvironError as e: printer.warning("could not load module '%s' correctly: " "Skipping..." % m) printer.debug(str(e)) if options.flex_alloc_tasks: printer.warning("`--flex-alloc-tasks' is deprecated and " "will be removed in the future; " "you should use --flex-alloc-nodes instead") options.flex_alloc_nodes = (options.flex_alloc_nodes or options.flex_alloc_tasks) options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle' # Act on checks success = True if options.list: # List matched checks list_checks(list(checks_matched), printer) elif options.list_detailed: # List matched checks with details list_checks(list(checks_matched), printer, detailed=True) elif options.run: # Setup the execution policy if options.exec_policy == 'serial': exec_policy = SerialExecutionPolicy() elif options.exec_policy == 'async': exec_policy = AsynchronousExecutionPolicy() else: # This should not happen, since choices are handled by # argparser printer.error("unknown execution policy `%s': Exiting...") sys.exit(1) exec_policy.skip_system_check = options.skip_system_check exec_policy.force_local = options.force_local exec_policy.strict_check = options.strict exec_policy.skip_sanity_check = options.skip_sanity_check exec_policy.skip_performance_check = options.skip_performance_check exec_policy.keep_stage_files = options.keep_stage_files try: errmsg = "invalid option for --flex-alloc-nodes: '{0}'" sched_flex_alloc_nodes = int(options.flex_alloc_nodes) if sched_flex_alloc_nodes <= 0: raise ConfigError(errmsg.format(options.flex_alloc_nodes)) except ValueError: if not options.flex_alloc_nodes.casefold() in {'idle', 'all'}: raise ConfigError(errmsg.format( options.flex_alloc_nodes)) from None sched_flex_alloc_nodes = options.flex_alloc_nodes exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes exec_policy.flex_alloc_nodes = options.flex_alloc_nodes exec_policy.sched_account = options.account exec_policy.sched_partition = options.partition exec_policy.sched_reservation = options.reservation exec_policy.sched_nodelist = options.nodelist exec_policy.sched_exclude_nodelist = options.exclude_nodes exec_policy.sched_options = options.job_options try: max_retries = int(options.max_retries) except ValueError: raise ConfigError('--max-retries is not a valid integer: %s' % max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: runner.runall(testcases) finally: # Print a retry report if we did any retries if runner.stats.failures(run=0): printer.info(runner.stats.retry_report()) # Print a failure report if we had failures in the last run if runner.stats.failures(): printer.info(runner.stats.failure_report()) success = False if options.performance_report: printer.info(runner.stats.performance_report()) else: printer.error("No action specified. Please specify `-l'/`-L' for " "listing or `-r' for running. " "Try `%s -h' for more options." % argparser.prog) sys.exit(1) if not success: sys.exit(1) sys.exit(0) except KeyboardInterrupt: sys.exit(1) except ReframeError as e: printer.error(str(e)) sys.exit(1) except (Exception, ReframeFatalError): printer.error(format_exception(*sys.exc_info())) sys.exit(1) finally: try: if options.save_log_files: logging.save_log_files(rt.resources.output_prefix) except OSError as e: printer.error('could not save log file: %s' % e) sys.exit(1)
def _setup_local_execution(): partition = rt.runtime().system.partition('login') environ = partition.environment('builtin-gcc') return partition, environ
def main(): # Setup command line options argparser = argparse.ArgumentParser() output_options = argparser.add_argument_group( 'Options controlling ReFrame output' ) locate_options = argparser.add_argument_group( 'Options for discovering checks' ) select_options = argparser.add_argument_group( 'Options for selecting checks' ) action_options = argparser.add_argument_group( 'Options controlling actions' ) run_options = argparser.add_argument_group( 'Options controlling the execution of checks' ) env_options = argparser.add_argument_group( 'Options controlling the ReFrame environment' ) misc_options = argparser.add_argument_group('Miscellaneous options') # Output directory options output_options.add_argument( '--prefix', action='store', metavar='DIR', help='Set general directory prefix to DIR', envvar='RFM_PREFIX', configvar='systems/prefix' ) output_options.add_argument( '-o', '--output', action='store', metavar='DIR', help='Set output directory prefix to DIR', envvar='RFM_OUTPUT_DIR', configvar='systems/outputdir' ) output_options.add_argument( '-s', '--stage', action='store', metavar='DIR', help='Set stage directory prefix to DIR', envvar='RFM_STAGE_DIR', configvar='systems/stagedir' ) output_options.add_argument( '--timestamp', action='store', nargs='?', const='', metavar='TIMEFMT', help=('Append a timestamp to the output and stage directory prefixes ' '(default: "%%FT%%T")'), envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs' ) output_options.add_argument( '--perflogdir', action='store', metavar='DIR', help=('Set performance log data directory prefix ' '(relevant only to the filelog log handler)'), envvar='RFM_PERFLOG_DIR', configvar='logging/handlers_perflog/filelog_basedir' ) output_options.add_argument( '--keep-stage-files', action='store_true', help='Keep stage directories even for successful checks', envvar='RFM_KEEP_STAGE_FILES', configvar='general/keep_stage_files' ) output_options.add_argument( '--dont-restage', action='store_false', dest='clean_stagedir', help='Reuse the test stage directory', envvar='RFM_CLEAN_STAGEDIR', configvar='general/clean_stagedir' ) output_options.add_argument( '--save-log-files', action='store_true', default=False, help='Save ReFrame log files to the output directory', envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) output_options.add_argument( '--report-file', action='store', metavar='FILE', help="Store JSON run report in FILE", envvar='RFM_REPORT_FILE', configvar='general/report_file' ) # Check discovery options locate_options.add_argument( '-c', '--checkpath', action='append', metavar='PATH', help="Add PATH to the check search path list", envvar='RFM_CHECK_SEARCH_PATH :', configvar='general/check_search_path' ) locate_options.add_argument( '-R', '--recursive', action='store_true', help='Search for checks in the search path recursively', envvar='RFM_CHECK_SEARCH_RECURSIVE', configvar='general/check_search_recursive' ) locate_options.add_argument( '--ignore-check-conflicts', action='store_true', help='Skip checks with conflicting names', envvar='RFM_IGNORE_CHECK_CONFLICTS', configvar='general/ignore_check_conflicts' ) # Select options select_options.add_argument( '-t', '--tag', action='append', dest='tags', metavar='PATTERN', default=[], help='Select checks with at least one tag matching PATTERN' ) select_options.add_argument( '-n', '--name', action='append', dest='names', default=[], metavar='PATTERN', help='Select checks whose name matches PATTERN' ) select_options.add_argument( '-x', '--exclude', action='append', dest='exclude_names', metavar='PATTERN', default=[], help='Exclude checks whose name matches PATTERN' ) select_options.add_argument( '-p', '--prgenv', action='append', default=[r'.*'], metavar='PATTERN', help=('Select checks with at least one ' 'programming environment matching PATTERN') ) select_options.add_argument( '--gpu-only', action='store_true', help='Select only GPU checks' ) select_options.add_argument( '--cpu-only', action='store_true', help='Select only CPU checks' ) # Action options action_options.add_argument( '-l', '--list', action='store_true', help='List the selected checks' ) action_options.add_argument( '-L', '--list-detailed', action='store_true', help='List the selected checks providing details for each test' ) action_options.add_argument( '-r', '--run', action='store_true', help='Run the selected checks' ) # Run options run_options.add_argument( '-A', '--account', action='store', help="Use ACCOUNT for submitting jobs (Slurm) " "*deprecated*, please use '-J account=ACCOUNT'") run_options.add_argument( '-P', '--partition', action='store', metavar='PART', help="Use PART for submitting jobs (Slurm/PBS/Torque) " "*deprecated*, please use '-J partition=PART' " "or '-J q=PART'") run_options.add_argument( '--reservation', action='store', metavar='RES', help="Use RES for submitting jobs (Slurm) " "*deprecated*, please use '-J reservation=RES'") run_options.add_argument( '--nodelist', action='store', help="Run checks on the selected list of nodes (Slurm) " "*deprecated*, please use '-J nodelist=NODELIST'") run_options.add_argument( '--exclude-nodes', action='store', metavar='NODELIST', help="Exclude the list of nodes from running checks (Slurm) " "*deprecated*, please use '-J exclude=NODELIST'") run_options.add_argument( '-J', '--job-option', action='append', metavar='OPT', dest='job_options', default=[], help='Pass option OPT to job scheduler' ) run_options.add_argument( '--force-local', action='store_true', help='Force local execution of checks' ) run_options.add_argument( '--skip-sanity-check', action='store_true', help='Skip sanity checking' ) run_options.add_argument( '--skip-performance-check', action='store_true', help='Skip performance checking' ) run_options.add_argument( '--strict', action='store_true', help='Enforce strict performance checking' ) run_options.add_argument( '--skip-system-check', action='store_true', help='Skip system check' ) run_options.add_argument( '--skip-prgenv-check', action='store_true', help='Skip programming environment check' ) run_options.add_argument( '--exec-policy', metavar='POLICY', action='store', choices=['async', 'serial'], default='async', help='Set the execution policy of ReFrame (default: "async")' ) run_options.add_argument( '--mode', action='store', help='Execution mode to use' ) run_options.add_argument( '--max-retries', metavar='NUM', action='store', default=0, help='Set the maximum number of times a failed regression test ' 'may be retried (default: 0)' ) run_options.add_argument( '--flex-alloc-nodes', action='store', dest='flex_alloc_nodes', metavar='{all|STATE|NUM}', default=None, help='Set strategy for the flexible node allocation (default: "idle").' ) env_options.add_argument( '-M', '--map-module', action='append', metavar='MAPPING', dest='module_mappings', default=[], help='Add a module mapping', envvar='RFM_MODULE_MAPPINGS ,', configvar='general/module_mappings' ) env_options.add_argument( '-m', '--module', action='append', default=[], metavar='MOD', dest='user_modules', help='Load module MOD before running any regression check', envvar='RFM_USER_MODULES ,', configvar='general/user_modules' ) env_options.add_argument( '--module-mappings', action='store', metavar='FILE', dest='module_map_file', help='Load module mappings from FILE', envvar='RFM_MODULE_MAP_FILE', configvar='general/module_map_file' ) env_options.add_argument( '-u', '--unload-module', action='append', metavar='MOD', dest='unload_modules', default=[], help='Unload module MOD before running any regression check', envvar='RFM_UNLOAD_MODULES ,', configvar='general/unload_modules' ) env_options.add_argument( '--purge-env', action='store_true', dest='purge_env', default=False, help='Unload all modules before running any regression check', envvar='RFM_PURGE_ENVIRONMENT', configvar='general/purge_environment' ) env_options.add_argument( '--non-default-craype', action='store_true', help='Test a non-default Cray Programming Environment', envvar='RFM_NON_DEFAULT_CRAYPE', configvar='general/non_default_craype' ) # Miscellaneous options misc_options.add_argument( '-C', '--config-file', action='store', dest='config_file', metavar='FILE', help='Set configuration file', envvar='RFM_CONFIG_FILE' ) misc_options.add_argument( '--nocolor', action='store_false', dest='colorize', help='Disable coloring of output', envvar='RFM_COLORIZE', configvar='general/colorize' ) misc_options.add_argument( '--failure-stats', action='store_true', help='Print failure statistics' ) misc_options.add_argument( '--performance-report', action='store_true', help='Print a report for performance tests' ) misc_options.add_argument( '--show-config', action='store', nargs='?', const='all', metavar='PARAM', help='Print the value of configuration parameter PARAM and exit' ) misc_options.add_argument( '--system', action='store', help='Load configuration for SYSTEM', envvar='RFM_SYSTEM' ) misc_options.add_argument( '--upgrade-config-file', action='store', metavar='OLD[:NEW]', help='Upgrade old configuration file to new syntax' ) misc_options.add_argument( '-V', '--version', action='version', version=os_ext.reframe_version() ) misc_options.add_argument( '-v', '--verbose', action='count', help='Increase verbosity level of output', envvar='RFM_VERBOSE', configvar='general/verbose' ) # Options not associated with command-line arguments argparser.add_argument( dest='graylog_server', envvar='RFM_GRAYLOG_ADDRESS', configvar='logging/handlers_perflog/graylog_address', help='Graylog server address' ) argparser.add_argument( dest='syslog_address', envvar='RFM_SYSLOG_ADDRESS', configvar='logging/handlers_perflog/syslog_address', help='Syslog server address' ) argparser.add_argument( dest='ignore_reqnodenotavail', envvar='RFM_IGNORE_REQNODENOTAVAIL', configvar='schedulers/ignore_reqnodenotavail', action='store_true', help='Graylog server address' ) argparser.add_argument( dest='use_login_shell', envvar='RFM_USE_LOGIN_SHELL', configvar='general/use_login_shell', action='store_true', help='Use a login shell for job scripts' ) if len(sys.argv) == 1: argparser.print_help() sys.exit(1) # Parse command line options = argparser.parse_args() # First configure logging with our generic configuration so as to be able # to print pretty messages; logging will be reconfigured by user's # configuration later site_config = config.load_config( os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py') ) site_config.select_subconfig('generic') options.update_config(site_config) logging.configure_logging(site_config) logging.getlogger().colorize = site_config.get('general/0/colorize') printer = PrettyPrinter() printer.colorize = site_config.get('general/0/colorize') printer.inc_verbosity(site_config.get('general/0/verbose')) if os.getenv('RFM_GRAYLOG_SERVER'): printer.warning( 'RFM_GRAYLOG_SERVER environment variable is deprecated; ' 'please use RFM_GRAYLOG_ADDRESS instead' ) os.environ['RFM_GRAYLOG_ADDRESS'] = os.getenv('RFM_GRAYLOG_SERVER') if options.upgrade_config_file is not None: old_config, *new_config = options.upgrade_config_file.split( ':', maxsplit=1) new_config = new_config[0] if new_config else None try: new_config = config.convert_old_config(old_config, new_config) except Exception as e: printer.error(f'could not convert file: {e}') sys.exit(1) printer.info( f'Conversion successful! ' f'The converted file can be found at {new_config!r}.' ) sys.exit(0) # Now configure ReFrame according to the user configuration file try: try: site_config = config.load_config(options.config_file) except ReframeDeprecationWarning as e: printer.warning(e) converted = config.convert_old_config(options.config_file) printer.warning( f"configuration file has been converted " f"to the new syntax here: '{converted}'" ) site_config = config.load_config(converted) site_config.validate() site_config.select_subconfig(options.system) for err in options.update_config(site_config): printer.warning(str(err)) # Update options from the selected execution mode if options.mode: mode_args = site_config.get(f'modes/@{options.mode}/options') # Parse the mode's options and reparse the command-line options = argparser.parse_args(mode_args) options = argparser.parse_args(namespace=options.cmd_options) options.update_config(site_config) logging.configure_logging(site_config) except (OSError, ConfigError) as e: printer.error(f'failed to load configuration: {e}') sys.exit(1) logging.getlogger().colorize = site_config.get('general/0/colorize') printer.colorize = site_config.get('general/0/colorize') printer.inc_verbosity(site_config.get('general/0/verbose')) try: runtime.init_runtime(site_config) except ConfigError as e: printer.error(f'failed to initialize runtime: {e}') sys.exit(1) rt = runtime.runtime() try: if site_config.get('general/0/module_map_file'): rt.modules_system.load_mapping_from_file( site_config.get('general/0/module_map_file') ) if site_config.get('general/0/module_mappings'): for m in site_config.get('general/0/module_mappings'): rt.modules_system.load_mapping(m) except (ConfigError, OSError) as e: printer.error('could not load module mappings: %s' % e) sys.exit(1) if (os_ext.samefile(rt.stage_prefix, rt.output_prefix) and not site_config.get('general/0/keep_stage_files')): printer.error("stage and output refer to the same directory; " "if this is on purpose, please use the " "'--keep-stage-files' option.") sys.exit(1) # Show configuration after everything is set up if options.show_config: config_param = options.show_config if config_param == 'all': printer.info(str(rt.site_config)) else: value = rt.get_option(config_param) if value is None: printer.error( f'no such configuration parameter found: {config_param}' ) else: printer.info(json.dumps(value, indent=2)) sys.exit(0) printer.debug(format_env(options.env_vars)) # Setup the check loader loader = RegressionCheckLoader( load_path=site_config.get('general/0/check_search_path'), recurse=site_config.get('general/0/check_search_recursive'), ignore_conflicts=site_config.get('general/0/ignore_check_conflicts') ) def print_infoline(param, value): param = param + ':' printer.info(f" {param.ljust(18)} {value}") session_info = { 'cmdline': ' '.join(sys.argv), 'config_file': rt.site_config.filename, 'data_version': '1.0', 'hostname': socket.gethostname(), 'prefix_output': rt.output_prefix, 'prefix_stage': rt.stage_prefix, 'user': os_ext.osuser(), 'version': os_ext.reframe_version(), 'workdir': os.getcwd(), } # Print command line printer.info(f"[ReFrame Setup]") print_infoline('version', session_info['version']) print_infoline('command', repr(session_info['cmdline'])) print_infoline( f"launched by", f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}" ) print_infoline('working directory', repr(session_info['workdir'])) print_infoline('settings file', f"{session_info['config_file']!r}") print_infoline('check search path', f"{'(R) ' if loader.recurse else ''}" f"{':'.join(loader.load_path)!r}") print_infoline('stage directory', repr(session_info['prefix_stage'])) print_infoline('output directory', repr(session_info['prefix_output'])) printer.info('') try: # Locate and load checks try: checks_found = loader.load_all() except OSError as e: raise ReframeError from e # Filter checks by name checks_matched = checks_found if options.exclude_names: for name in options.exclude_names: checks_matched = filter(filters.have_not_name(name), checks_matched) if options.names: checks_matched = filter(filters.have_name('|'.join(options.names)), checks_matched) # Filter checks by tags for tag in options.tags: checks_matched = filter(filters.have_tag(tag), checks_matched) # Filter checks by prgenv if not options.skip_prgenv_check: for prgenv in options.prgenv: checks_matched = filter(filters.have_prgenv(prgenv), checks_matched) # Filter checks by system if not options.skip_system_check: checks_matched = filter( filters.have_partition(rt.system.partitions), checks_matched) # Filter checks further if options.gpu_only and options.cpu_only: printer.error("options `--gpu-only' and `--cpu-only' " "are mutually exclusive") sys.exit(1) if options.gpu_only: checks_matched = filter(filters.have_gpu_only(), checks_matched) elif options.cpu_only: checks_matched = filter(filters.have_cpu_only(), checks_matched) # Determine the allowed programming environments allowed_environs = {e.name for env_patt in options.prgenv for p in rt.system.partitions for e in p.environs if re.match(env_patt, e.name)} # Generate the test cases, validate dependencies and sort them checks_matched = list(checks_matched) testcases = generate_testcases(checks_matched, options.skip_system_check, options.skip_prgenv_check, allowed_environs) testgraph = dependency.build_deps(testcases) dependency.validate_deps(testgraph) testcases = dependency.toposort(testgraph) # Manipulate ReFrame's environment if site_config.get('general/0/purge_environment'): rt.modules_system.unload_all() else: for m in site_config.get('general/0/unload_modules'): rt.modules_system.unload_module(m) # Load the environment for the current system try: runtime.loadenv(rt.system.preload_environ) except EnvironError as e: printer.error("failed to load current system's environment; " "please check your configuration") printer.debug(str(e)) raise for m in site_config.get('general/0/user_modules'): try: rt.modules_system.load_module(m, force=True) except EnvironError as e: printer.warning("could not load module '%s' correctly: " "Skipping..." % m) printer.debug(str(e)) options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle' if options.account: printer.warning(f"`--account' is deprecated and " f"will be removed in the future; you should " f"use `-J account={options.account}'") if options.partition: printer.warning(f"`--partition' is deprecated and " f"will be removed in the future; you should " f"use `-J partition={options.partition}' " f"or `-J q={options.partition}' depending on your " f"scheduler") if options.reservation: printer.warning(f"`--reservation' is deprecated and " f"will be removed in the future; you should " f"use `-J reservation={options.reservation}'") if options.nodelist: printer.warning(f"`--nodelist' is deprecated and " f"will be removed in the future; you should " f"use `-J nodelist={options.nodelist}'") if options.exclude_nodes: printer.warning(f"`--exclude-nodes' is deprecated and " f"will be removed in the future; you should " f"use `-J exclude={options.exclude_nodes}'") # Act on checks success = True if options.list: # List matched checks list_checks(list(checks_matched), printer) elif options.list_detailed: # List matched checks with details list_checks(list(checks_matched), printer, detailed=True) elif options.run: # Setup the execution policy if options.exec_policy == 'serial': exec_policy = SerialExecutionPolicy() elif options.exec_policy == 'async': exec_policy = AsynchronousExecutionPolicy() else: # This should not happen, since choices are handled by # argparser printer.error("unknown execution policy `%s': Exiting...") sys.exit(1) exec_policy.skip_system_check = options.skip_system_check exec_policy.force_local = options.force_local exec_policy.strict_check = options.strict exec_policy.skip_sanity_check = options.skip_sanity_check exec_policy.skip_performance_check = options.skip_performance_check exec_policy.keep_stage_files = site_config.get( 'general/0/keep_stage_files' ) try: errmsg = "invalid option for --flex-alloc-nodes: '{0}'" sched_flex_alloc_nodes = int(options.flex_alloc_nodes) if sched_flex_alloc_nodes <= 0: raise ConfigError(errmsg.format(options.flex_alloc_nodes)) except ValueError: sched_flex_alloc_nodes = options.flex_alloc_nodes exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes exec_policy.flex_alloc_nodes = options.flex_alloc_nodes exec_policy.sched_account = options.account exec_policy.sched_partition = options.partition exec_policy.sched_reservation = options.reservation exec_policy.sched_nodelist = options.nodelist exec_policy.sched_exclude_nodelist = options.exclude_nodes parsed_job_options = [] for opt in options.job_options: if opt.startswith('-') or opt.startswith('#'): parsed_job_options.append(opt) elif len(opt) == 1: parsed_job_options.append(f'-{opt}') else: parsed_job_options.append(f'--{opt}') exec_policy.sched_options = parsed_job_options try: max_retries = int(options.max_retries) except ValueError: raise ConfigError('--max-retries is not a valid integer: %s' % max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: time_start = time.time() session_info['time_start'] = time.strftime( '%FT%T%z', time.localtime(time_start), ) runner.runall(testcases) finally: time_end = time.time() session_info['time_end'] = time.strftime( '%FT%T%z', time.localtime(time_end) ) session_info['time_elapsed'] = time_end - time_start # Print a retry report if we did any retries if runner.stats.failures(run=0): printer.info(runner.stats.retry_report()) # Print a failure report if we had failures in the last run if runner.stats.failures(): printer.info(runner.stats.failure_report()) success = False if options.failure_stats: printer.info(runner.stats.failure_stats()) if options.performance_report: printer.info(runner.stats.performance_report()) # Generate the report for this session report_file = os.path.normpath( os_ext.expandvars(rt.get_option('general/0/report_file')) ) basedir = os.path.dirname(report_file) if basedir: os.makedirs(basedir, exist_ok=True) # Build final JSON report run_stats = runner.stats.json() session_info.update({ 'num_cases': run_stats[0]['num_cases'], 'num_failures': run_stats[-1]['num_failures'] }) json_report = { 'session_info': session_info, 'runs': run_stats } report_file = generate_report_filename(report_file) try: with open(report_file, 'w') as fp: json.dump(json_report, fp, indent=2) except OSError as e: printer.warning( f'failed to generate report in {report_file!r}: {e}' ) else: printer.error("No action specified. Please specify `-l'/`-L' for " "listing or `-r' for running. " "Try `%s -h' for more options." % argparser.prog) sys.exit(1) if not success: sys.exit(1) sys.exit(0) except KeyboardInterrupt: sys.exit(1) except ReframeError as e: printer.error(str(e)) sys.exit(1) except (Exception, ReframeFatalError): printer.error(format_exception(*sys.exc_info())) sys.exit(1) finally: try: if site_config.get('general/0/save_log_files'): logging.save_log_files(rt.output_prefix) except OSError as e: printer.error('could not save log file: %s' % e) sys.exit(1)
def _emit_gitlab_pipeline(testcases): config = runtime.runtime().site_config # Collect the necessary ReFrame invariants program = 'reframe' prefix = 'rfm-stage/${CI_COMMIT_SHORT_SHA}' checkpath = config.get('general/0/check_search_path') recurse = config.get('general/0/check_search_recursive') verbosity = 'v' * config.get('general/0/verbose') def rfm_command(testcase): if config.filename != '<builtin>': config_opt = f'-C {config.filename}' else: config_opt = '' report_file = f'{testcase.check.unique_name}-report.json' if testcase.level: restore_files = ','.join(f'{t.check.unique_name}-report.json' for t in tc.deps) else: restore_files = None return ' '.join([ program, f'--prefix={prefix}', config_opt, f'{" ".join("-c " + c for c in checkpath)}', f'-R' if recurse else '', f'--report-file={report_file}', f'--restore-session={restore_files}' if restore_files else '', f'--report-junit={testcase.check.unique_name}-report.xml', f'{"".join("-" + verbosity)}' if verbosity else '', '-n', f"'^{testcase.check.unique_name}$'", '-r' ]) max_level = 0 # We need the maximum level to generate the stages section json = { 'cache': { 'key': '${CI_COMMIT_REF_SLUG}', 'paths': ['rfm-stage/${CI_COMMIT_SHORT_SHA}'] }, 'stages': [] } # Name of the image used for CI. If user does not explicitly provide # image keyword on the top of CI script, this variable does not exist image_name = os.getenv('CI_JOB_IMAGE') if image_name: json['image'] = image_name for tc in testcases: json[f'{tc.check.unique_name}'] = { 'stage': f'rfm-stage-{tc.level}', 'script': [rfm_command(tc)], 'artifacts': { 'paths': [f'{tc.check.unique_name}-report.json'] }, 'needs': [t.check.unique_name for t in tc.deps] } max_level = max(max_level, tc.level) json['stages'] = [f'rfm-stage-{m}' for m in range(max_level + 1)] return json