def ctx_sys(fixture_exec_ctx):
yield rt.runtime().system
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
os_ext.rmtree('.rfm_testing', ignore_errors=True)
def main():
# Setup command line options
argparser = argparse.ArgumentParser()
output_options = argparser.add_argument_group(
'Options controlling ReFrame output')
locate_options = argparser.add_argument_group(
'Options for discovering checks')
select_options = argparser.add_argument_group(
'Options for selecting checks')
action_options = argparser.add_argument_group(
'Options controlling actions')
run_options = argparser.add_argument_group(
'Options controlling the execution of checks')
env_options = argparser.add_argument_group(
'Options controlling the ReFrame environment')
misc_options = argparser.add_argument_group('Miscellaneous options')
# Output directory options
output_options.add_argument('--prefix',
action='store',
metavar='DIR',
help='Set general directory prefix to DIR',
envvar='RFM_PREFIX',
configvar='systems/prefix')
output_options.add_argument('-o',
'--output',
action='store',
metavar='DIR',
help='Set output directory prefix to DIR',
envvar='RFM_OUTPUT_DIR',
configvar='systems/outputdir')
output_options.add_argument('-s',
'--stage',
action='store',
metavar='DIR',
help='Set stage directory prefix to DIR',
envvar='RFM_STAGE_DIR',
configvar='systems/stagedir')
output_options.add_argument(
'--timestamp',
action='store',
nargs='?',
const='',
metavar='TIMEFMT',
help=('Append a timestamp to the output and stage directory prefixes '
'(default: "%%FT%%T")'),
envvar='RFM_TIMESTAMP_DIRS',
configvar='general/timestamp_dirs')
output_options.add_argument(
'--perflogdir',
action='store',
metavar='DIR',
help=('Set performance log data directory prefix '
'(relevant only to the filelog log handler)'),
envvar='RFM_PERFLOG_DIR',
configvar='logging/handlers_perflog/filelog_basedir')
output_options.add_argument(
'--keep-stage-files',
action='store_true',
help='Keep stage directories even for successful checks',
envvar='RFM_KEEP_STAGE_FILES',
configvar='general/keep_stage_files')
output_options.add_argument('--dont-restage',
action='store_false',
dest='clean_stagedir',
help='Reuse the test stage directory',
envvar='RFM_CLEAN_STAGEDIR',
configvar='general/clean_stagedir')
output_options.add_argument(
'--save-log-files',
action='store_true',
default=False,
help='Save ReFrame log files to the output directory',
envvar='RFM_SAVE_LOG_FILES',
configvar='general/save_log_files')
output_options.add_argument('--report-file',
action='store',
metavar='FILE',
help="Store JSON run report in FILE",
envvar='RFM_REPORT_FILE',
configvar='general/report_file')
output_options.add_argument('--report-junit',
action='store',
metavar='FILE',
help="Store a JUnit report in FILE",
envvar='RFM_REPORT_JUNIT',
configvar='general/report_junit')
# Check discovery options
locate_options.add_argument('-c',
'--checkpath',
action='append',
metavar='PATH',
help="Add PATH to the check search path list",
envvar='RFM_CHECK_SEARCH_PATH :',
configvar='general/check_search_path')
locate_options.add_argument(
'-R',
'--recursive',
action='store_true',
help='Search for checks in the search path recursively',
envvar='RFM_CHECK_SEARCH_RECURSIVE',
configvar='general/check_search_recursive')
locate_options.add_argument(
'--ignore-check-conflicts',
action='store_true',
help=('Skip checks with conflicting names '
'(this option is deprecated and has no effect)'),
envvar='RFM_IGNORE_CHECK_CONFLICTS',
configvar='general/ignore_check_conflicts')
# Select options
select_options.add_argument(
'-t',
'--tag',
action='append',
dest='tags',
metavar='PATTERN',
default=[],
help='Select checks with at least one tag matching PATTERN')
select_options.add_argument(
'-n',
'--name',
action='append',
dest='names',
default=[],
metavar='PATTERN',
help='Select checks whose name matches PATTERN')
select_options.add_argument(
'-x',
'--exclude',
action='append',
dest='exclude_names',
metavar='PATTERN',
default=[],
help='Exclude checks whose name matches PATTERN')
select_options.add_argument(
'-p',
'--prgenv',
action='append',
default=[r'.*'],
metavar='PATTERN',
help=('Select checks with at least one '
'programming environment matching PATTERN'))
select_options.add_argument(
'--failed',
action='store_true',
help="Select failed test cases (only when '--restore-session' is used)"
)
select_options.add_argument('--gpu-only',
action='store_true',
help='Select only GPU checks')
select_options.add_argument('--cpu-only',
action='store_true',
help='Select only CPU checks')
# Action options
action_options.add_argument('-l',
'--list',
action='store_true',
help='List the selected checks')
action_options.add_argument(
'-L',
'--list-detailed',
action='store_true',
help='List the selected checks providing details for each test')
action_options.add_argument(
'--list-tags',
action='store_true',
help='List the unique tags found in the selected tests and exit')
action_options.add_argument('-r',
'--run',
action='store_true',
help='Run the selected checks')
action_options.add_argument(
'--ci-generate',
action='store',
metavar='FILE',
help=('Generate into FILE a Gitlab CI pipeline '
'for the selected tests and exit'),
)
# Run options
run_options.add_argument('-J',
'--job-option',
action='append',
metavar='OPT',
dest='job_options',
default=[],
help='Pass option OPT to job scheduler')
run_options.add_argument('--force-local',
action='store_true',
help='Force local execution of checks')
run_options.add_argument('--skip-sanity-check',
action='store_true',
help='Skip sanity checking')
run_options.add_argument('--skip-performance-check',
action='store_true',
help='Skip performance checking')
run_options.add_argument('--strict',
action='store_true',
help='Enforce strict performance checking')
run_options.add_argument('--skip-system-check',
action='store_true',
help='Skip system check')
run_options.add_argument('--skip-prgenv-check',
action='store_true',
help='Skip programming environment check')
run_options.add_argument('-S',
'--setvar',
action='append',
metavar='[TEST.]VAR=VAL',
dest='vars',
default=[],
help=('Set test variable VAR to VAL in all tests '
'or optionally in TEST only'))
run_options.add_argument(
'--exec-policy',
metavar='POLICY',
action='store',
choices=['async', 'serial'],
default='async',
help='Set the execution policy of ReFrame (default: "async")')
run_options.add_argument('--mode',
action='store',
help='Execution mode to use')
run_options.add_argument(
'--max-retries',
metavar='NUM',
action='store',
default=0,
help='Set the maximum number of times a failed regression test '
'may be retried (default: 0)')
run_options.add_argument('--maxfail',
metavar='NUM',
action='store',
default=sys.maxsize,
help='Exit after first NUM failures')
run_options.add_argument('--restore-session',
action='store',
nargs='?',
const='',
metavar='REPORT',
help='Restore a testing session from REPORT file')
run_options.add_argument(
'--flex-alloc-nodes',
action='store',
dest='flex_alloc_nodes',
metavar='{all|STATE|NUM}',
default=None,
help='Set strategy for the flexible node allocation (default: "idle").'
)
run_options.add_argument('--disable-hook',
action='append',
metavar='NAME',
dest='hooks',
default=[],
help='Disable a pipeline hook for this run')
# Environment options
env_options.add_argument('-M',
'--map-module',
action='append',
metavar='MAPPING',
dest='module_mappings',
default=[],
help='Add a module mapping',
envvar='RFM_MODULE_MAPPINGS ,',
configvar='general/module_mappings')
env_options.add_argument(
'-m',
'--module',
action='append',
default=[],
metavar='MOD',
dest='user_modules',
help='Load module MOD before running any regression check',
envvar='RFM_USER_MODULES ,',
configvar='general/user_modules')
env_options.add_argument('--module-mappings',
action='store',
metavar='FILE',
dest='module_map_file',
help='Load module mappings from FILE',
envvar='RFM_MODULE_MAP_FILE',
configvar='general/module_map_file')
env_options.add_argument(
'-u',
'--unload-module',
action='append',
metavar='MOD',
dest='unload_modules',
default=[],
help='Unload module MOD before running any regression check',
envvar='RFM_UNLOAD_MODULES ,',
configvar='general/unload_modules')
env_options.add_argument(
'--module-path',
action='append',
metavar='PATH',
dest='module_paths',
default=[],
help='(Un)use module path PATH before running any regression check',
)
env_options.add_argument(
'--purge-env',
action='store_true',
dest='purge_env',
default=False,
help='Unload all modules before running any regression check',
envvar='RFM_PURGE_ENVIRONMENT',
configvar='general/purge_environment')
env_options.add_argument(
'--non-default-craype',
action='store_true',
help='Test a non-default Cray Programming Environment',
envvar='RFM_NON_DEFAULT_CRAYPE',
configvar='general/non_default_craype')
# Miscellaneous options
misc_options.add_argument('-C',
'--config-file',
action='store',
dest='config_file',
metavar='FILE',
help='Set configuration file',
envvar='RFM_CONFIG_FILE')
misc_options.add_argument('--nocolor',
action='store_false',
dest='colorize',
help='Disable coloring of output',
envvar='RFM_COLORIZE',
configvar='general/colorize')
misc_options.add_argument('--failure-stats',
action='store_true',
help='Print failure statistics')
misc_options.add_argument('--performance-report',
action='store_true',
help='Print a report for performance tests')
misc_options.add_argument(
'--show-config',
action='store',
nargs='?',
const='all',
metavar='PARAM',
help='Print the value of configuration parameter PARAM and exit')
misc_options.add_argument('--system',
action='store',
help='Load configuration for SYSTEM',
envvar='RFM_SYSTEM')
misc_options.add_argument('--detect-host-topology',
action='store',
nargs='?',
const='-',
help='Detect the local host topology and exit')
misc_options.add_argument(
'--upgrade-config-file',
action='store',
metavar='OLD[:NEW]',
help='Upgrade ReFrame 2.x configuration file to ReFrame 3.x syntax')
misc_options.add_argument('-V',
'--version',
action='version',
version=osext.reframe_version())
misc_options.add_argument('-v',
'--verbose',
action='count',
help='Increase verbosity level of output',
envvar='RFM_VERBOSE',
configvar='general/verbose')
# Options not associated with command-line arguments
argparser.add_argument(
dest='graylog_server',
envvar='RFM_GRAYLOG_ADDRESS',
configvar='logging/handlers_perflog/graylog_address',
help='Graylog server address')
argparser.add_argument(dest='syslog_address',
envvar='RFM_SYSLOG_ADDRESS',
configvar='logging/handlers_perflog/syslog_address',
help='Syslog server address')
argparser.add_argument(dest='ignore_reqnodenotavail',
envvar='RFM_IGNORE_REQNODENOTAVAIL',
configvar='schedulers/ignore_reqnodenotavail',
action='store_true',
help='Graylog server address')
argparser.add_argument(dest='use_login_shell',
envvar='RFM_USE_LOGIN_SHELL',
configvar='general/use_login_shell',
action='store_true',
help='Use a login shell for job scripts')
argparser.add_argument(dest='resolve_module_conflicts',
envvar='RFM_RESOLVE_MODULE_CONFLICTS',
configvar='general/resolve_module_conflicts',
action='store_true',
help='Resolve module conflicts automatically')
argparser.add_argument(dest='httpjson_url',
envvar='RFM_HTTPJSON_URL',
configvar='logging/handlers_perflog/httpjson_url',
help='URL of HTTP server accepting JSON logs')
argparser.add_argument(dest='remote_detect',
envvar='RFM_REMOTE_DETECT',
configvar='general/remote_detect',
action='store_true',
help='Detect remote system topology')
argparser.add_argument(
dest='remote_workdir',
envvar='RFM_REMOTE_WORKDIR',
configvar='general/remote_workdir',
action='store',
help='Working directory for launching ReFrame remotely')
# Parse command line
options = argparser.parse_args()
if len(sys.argv) == 1:
argparser.print_help()
sys.exit(1)
# First configure logging with our generic configuration so as to be able
# to print pretty messages; logging will be reconfigured by user's
# configuration later
site_config = config.load_config(
os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py'))
site_config.select_subconfig('generic')
options.update_config(site_config)
logging.configure_logging(site_config)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer = PrettyPrinter()
printer.colorize = site_config.get('general/0/colorize')
printer.inc_verbosity(site_config.get('general/0/verbose'))
if os.getenv('RFM_GRAYLOG_SERVER'):
printer.warning(
'RFM_GRAYLOG_SERVER environment variable is deprecated; '
'please use RFM_GRAYLOG_ADDRESS instead')
os.environ['RFM_GRAYLOG_ADDRESS'] = os.getenv('RFM_GRAYLOG_SERVER')
if options.upgrade_config_file is not None:
old_config, *new_config = options.upgrade_config_file.split(':',
maxsplit=1)
new_config = new_config[0] if new_config else None
try:
new_config = config.convert_old_config(old_config, new_config)
except Exception as e:
printer.error(f'could not convert file: {e}')
sys.exit(1)
printer.info(f'Conversion successful! '
f'The converted file can be found at {new_config!r}.')
sys.exit(0)
# Now configure ReFrame according to the user configuration file
try:
try:
printer.debug('Loading user configuration')
site_config = config.load_config(options.config_file)
except warnings.ReframeDeprecationWarning as e:
printer.warning(e)
converted = config.convert_old_config(options.config_file)
printer.warning(f"configuration file has been converted "
f"to the new syntax here: '{converted}'")
site_config = config.load_config(converted)
site_config.validate()
# We ignore errors about unresolved sections or configuration
# parameters here, because they might be defined at the individual
# partition level and will be caught when we will instantiating
# internally the system and partitions later on.
site_config.select_subconfig(options.system,
ignore_resolve_errors=True)
for err in options.update_config(site_config):
printer.warning(str(err))
# Update options from the selected execution mode
if options.mode:
mode_args = site_config.get(f'modes/@{options.mode}/options')
# We lexically split the mode options, because otherwise spaces
# will be treated as part of the option argument; see GH bug #1554
mode_args = list(
itertools.chain.from_iterable(
shlex.split(m) for m in mode_args))
# Parse the mode's options and reparse the command-line
options = argparser.parse_args(mode_args)
options = argparser.parse_args(namespace=options.cmd_options)
options.update_config(site_config)
logging.configure_logging(site_config)
except (OSError, errors.ConfigError) as e:
printer.error(f'failed to load configuration: {e}')
printer.error(logfiles_message())
sys.exit(1)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer.colorize = site_config.get('general/0/colorize')
printer.inc_verbosity(site_config.get('general/0/verbose'))
try:
printer.debug('Initializing runtime')
runtime.init_runtime(site_config)
except errors.ConfigError as e:
printer.error(f'failed to initialize runtime: {e}')
printer.error(logfiles_message())
sys.exit(1)
if site_config.get('general/0/ignore_check_conflicts'):
logging.getlogger().warning(
"the 'ignore_check_conflicts' option is deprecated "
"and will be removed in the future")
rt = runtime.runtime()
autodetect.detect_topology()
try:
if site_config.get('general/0/module_map_file'):
rt.modules_system.load_mapping_from_file(
site_config.get('general/0/module_map_file'))
if site_config.get('general/0/module_mappings'):
for m in site_config.get('general/0/module_mappings'):
rt.modules_system.load_mapping(m)
except (errors.ConfigError, OSError) as e:
printer.error('could not load module mappings: %s' % e)
sys.exit(1)
if (osext.samefile(rt.stage_prefix, rt.output_prefix)
and not site_config.get('general/0/keep_stage_files')):
printer.error("stage and output refer to the same directory; "
"if this is on purpose, please use the "
"'--keep-stage-files' option.")
printer.error(logfiles_message())
sys.exit(1)
# Show configuration after everything is set up
if options.show_config:
config_param = options.show_config
if config_param == 'all':
printer.info(str(rt.site_config))
else:
value = rt.get_option(config_param)
if value is None:
printer.error(
f'no such configuration parameter found: {config_param}')
else:
printer.info(json.dumps(value, indent=2))
sys.exit(0)
if options.detect_host_topology:
from reframe.utility.cpuinfo import cpuinfo
topofile = options.detect_host_topology
if topofile == '-':
json.dump(cpuinfo(), sys.stdout, indent=2)
sys.stdout.write('\n')
else:
try:
with open(topofile, 'w') as fp:
json.dump(cpuinfo(), fp, indent=2)
fp.write('\n')
except OSError as e:
getlogger().error(
f'could not write topology file: {topofile!r}')
sys.exit(1)
sys.exit(0)
printer.debug(format_env(options.env_vars))
# Setup the check loader
if options.restore_session is not None:
# We need to load the failed checks only from a list of reports
if options.restore_session:
filenames = options.restore_session.split(',')
else:
filenames = [
runreport.next_report_filename(osext.expandvars(
site_config.get('general/0/report_file')),
new=False)
]
report = runreport.load_report(*filenames)
check_search_path = list(report.slice('filename', unique=True))
check_search_recursive = False
# If `-c` or `-R` are passed explicitly outside the configuration
# file, override the values set from the report file
if site_config.is_sticky_option('general/check_search_path'):
printer.warning(
'Ignoring check search path set in the report file: '
'search path set explicitly in the command-line or '
'the environment')
check_search_path = site_config.get('general/0/check_search_path')
if site_config.is_sticky_option('general/check_search_recursive'):
printer.warning(
'Ignoring check search recursive option from the report file: '
'option set explicitly in the command-line or the environment')
check_search_recursive = site_config.get(
'general/0/check_search_recursive')
else:
check_search_recursive = site_config.get(
'general/0/check_search_recursive')
check_search_path = site_config.get('general/0/check_search_path')
# Collect any variables set from the command line
external_vars = {}
for expr in options.vars:
try:
lhs, rhs = expr.split('=', maxsplit=1)
except ValueError:
printer.warning(
f'invalid test variable assignment: {expr!r}; skipping')
else:
external_vars[lhs] = rhs
loader = RegressionCheckLoader(check_search_path, check_search_recursive,
external_vars)
def print_infoline(param, value):
param = param + ':'
printer.info(f" {param.ljust(18)} {value}")
session_info = {
'cmdline': ' '.join(sys.argv),
'config_file': rt.site_config.filename,
'data_version': runreport.DATA_VERSION,
'hostname': socket.gethostname(),
'prefix_output': rt.output_prefix,
'prefix_stage': rt.stage_prefix,
'user': osext.osuser(),
'version': osext.reframe_version(),
'workdir': os.getcwd(),
}
# Print command line
printer.info(f"[ReFrame Setup]")
print_infoline('version', session_info['version'])
print_infoline('command', repr(session_info['cmdline']))
print_infoline(
f"launched by",
f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}")
print_infoline('working directory', repr(session_info['workdir']))
print_infoline('settings file', f"{session_info['config_file']!r}")
print_infoline(
'check search path', f"{'(R) ' if loader.recurse else ''}"
f"{':'.join(loader.load_path)!r}")
print_infoline('stage directory', repr(session_info['prefix_stage']))
print_infoline('output directory', repr(session_info['prefix_output']))
printer.info('')
try:
# Locate and load checks
checks_found = loader.load_all()
printer.verbose(f'Loaded {len(checks_found)} test(s)')
# Generate all possible test cases first; we will need them for
# resolving dependencies after filtering
# Determine the allowed programming environments
allowed_environs = {
e.name
for env_patt in options.prgenv for p in rt.system.partitions
for e in p.environs if re.match(env_patt, e.name)
}
testcases_all = generate_testcases(checks_found,
options.skip_system_check,
options.skip_prgenv_check,
allowed_environs)
testcases = testcases_all
printer.verbose(f'Generated {len(testcases)} test case(s)')
# Filter test cases by name
if options.exclude_names:
for name in options.exclude_names:
testcases = filter(filters.have_not_name(name), testcases)
if options.names:
testcases = filter(filters.have_name('|'.join(options.names)),
testcases)
testcases = list(testcases)
printer.verbose(
f'Filtering test cases(s) by name: {len(testcases)} remaining')
# Filter test cases by tags
for tag in options.tags:
testcases = filter(filters.have_tag(tag), testcases)
testcases = list(testcases)
printer.verbose(
f'Filtering test cases(s) by tags: {len(testcases)} remaining')
# Filter test cases further
if options.gpu_only and options.cpu_only:
printer.error("options `--gpu-only' and `--cpu-only' "
"are mutually exclusive")
sys.exit(1)
if options.gpu_only:
testcases = filter(filters.have_gpu_only(), testcases)
elif options.cpu_only:
testcases = filter(filters.have_cpu_only(), testcases)
testcases = list(testcases)
printer.verbose(f'Filtering test cases(s) by other attributes: '
f'{len(testcases)} remaining')
# Filter in failed cases
if options.failed:
if options.restore_session is None:
printer.error(
"the option '--failed' can only be used "
"in combination with the '--restore-session' option")
sys.exit(1)
def _case_failed(t):
rec = report.case(*t)
if not rec:
return False
return (rec['result'] == 'failure'
or rec['result'] == 'aborted')
testcases = list(filter(_case_failed, testcases))
printer.verbose(f'Filtering successful test case(s): '
f'{len(testcases)} remaining')
# Prepare for running
printer.debug('Building and validating the full test DAG')
testgraph, skipped_cases = dependencies.build_deps(testcases_all)
if skipped_cases:
# Some cases were skipped, so adjust testcases
testcases = list(set(testcases) - set(skipped_cases))
printer.verbose(
f'Filtering test case(s) due to unresolved dependencies: '
f'{len(testcases)} remaining')
dependencies.validate_deps(testgraph)
printer.debug('Full test DAG:')
printer.debug(dependencies.format_deps(testgraph))
restored_cases = []
if len(testcases) != len(testcases_all):
testgraph = dependencies.prune_deps(
testgraph,
testcases,
max_depth=1 if options.restore_session is not None else None)
printer.debug('Pruned test DAG')
printer.debug(dependencies.format_deps(testgraph))
if options.restore_session is not None:
testgraph, restored_cases = report.restore_dangling(testgraph)
testcases = dependencies.toposort(testgraph,
is_subgraph=options.restore_session
is not None)
printer.verbose(f'Final number of test cases: {len(testcases)}')
# Disable hooks
for tc in testcases:
for h in options.hooks:
tc.check.disable_hook(h)
# Act on checks
if options.list or options.list_detailed:
list_checks(testcases, printer, options.list_detailed)
sys.exit(0)
if options.list_tags:
list_tags(testcases, printer)
sys.exit(0)
if options.ci_generate:
list_checks(testcases, printer)
printer.info('[Generate CI]')
with open(options.ci_generate, 'wt') as fp:
ci.emit_pipeline(fp, testcases)
printer.info(f' Gitlab pipeline generated successfully '
f'in {options.ci_generate!r}.\n')
sys.exit(0)
if not options.run:
printer.error("No action option specified. Available options:\n"
" - `-l'/`-L' for listing\n"
" - `-r' for running\n"
" - `--list-tags' for listing unique test tags\n"
" - `--ci-generate' for generating a CI pipeline\n"
f"Try `{argparser.prog} -h' for more options.")
sys.exit(1)
# Manipulate ReFrame's environment
if site_config.get('general/0/purge_environment'):
rt.modules_system.unload_all()
else:
for m in site_config.get('general/0/unload_modules'):
rt.modules_system.unload_module(**m)
# Load the environment for the current system
try:
printer.debug(f'Loading environment for current system')
runtime.loadenv(rt.system.preload_environ)
except errors.EnvironError as e:
printer.error("failed to load current system's environment; "
"please check your configuration")
printer.debug(str(e))
raise
def module_use(*paths):
try:
rt.modules_system.searchpath_add(*paths)
except errors.EnvironError as e:
printer.warning(f'could not add module paths correctly')
printer.debug(str(e))
def module_unuse(*paths):
try:
rt.modules_system.searchpath_remove(*paths)
except errors.EnvironError as e:
printer.warning(f'could not remove module paths correctly')
printer.debug(str(e))
printer.debug('(Un)using module paths from command line')
module_paths = {}
for d in options.module_paths:
if d.startswith('-'):
module_paths.setdefault('-', [])
module_paths['-'].append(d[1:])
elif d.startswith('+'):
module_paths.setdefault('+', [])
module_paths['+'].append(d[1:])
else:
module_paths.setdefault('x', [])
module_paths['x'].append(d)
for op, paths in module_paths.items():
if op == '+':
module_use(*paths)
elif op == '-':
module_unuse(*paths)
else:
# First empty the current module path in a portable way
searchpath = [p for p in rt.modules_system.searchpath if p]
if searchpath:
rt.modules_system.searchpath_remove(*searchpath)
# Treat `A:B` syntax as well in this case
paths = itertools.chain(*(p.split(':') for p in paths))
module_use(*paths)
printer.debug('Loading user modules from command line')
for m in site_config.get('general/0/user_modules'):
try:
rt.modules_system.load_module(**m, force=True)
except errors.EnvironError as e:
printer.warning(
f'could not load module {m["name"]!r} correctly; '
f'skipping...')
printer.debug(str(e))
options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle'
# Run the tests
# Setup the execution policy
if options.exec_policy == 'serial':
exec_policy = SerialExecutionPolicy()
elif options.exec_policy == 'async':
exec_policy = AsynchronousExecutionPolicy()
else:
# This should not happen, since choices are handled by
# argparser
printer.error("unknown execution policy `%s': Exiting...")
sys.exit(1)
exec_policy.skip_system_check = options.skip_system_check
exec_policy.force_local = options.force_local
exec_policy.strict_check = options.strict
exec_policy.skip_sanity_check = options.skip_sanity_check
exec_policy.skip_performance_check = options.skip_performance_check
exec_policy.keep_stage_files = site_config.get(
'general/0/keep_stage_files')
try:
errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
if sched_flex_alloc_nodes <= 0:
raise errors.ConfigError(
errmsg.format(options.flex_alloc_nodes))
except ValueError:
sched_flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes
parsed_job_options = []
for opt in options.job_options:
opt_split = opt.split('=', maxsplit=1)
optstr = opt_split[0]
valstr = opt_split[1] if len(opt_split) > 1 else ''
if opt.startswith('-') or opt.startswith('#'):
parsed_job_options.append(opt)
elif len(optstr) == 1:
parsed_job_options.append(f'-{optstr} {valstr}')
else:
parsed_job_options.append(f'--{optstr} {valstr}')
exec_policy.sched_options = parsed_job_options
try:
max_retries = int(options.max_retries)
except ValueError:
raise errors.ConfigError(
f'--max-retries is not a valid integer: {max_retries}'
) from None
try:
max_failures = int(options.maxfail)
if max_failures < 0:
raise errors.ConfigError(
f'--maxfail should be a non-negative integer: '
f'{options.maxfail!r}')
except ValueError:
raise errors.ConfigError(
f'--maxfail is not a valid integer: {options.maxfail!r}'
) from None
runner = Runner(exec_policy, printer, max_retries, max_failures)
try:
time_start = time.time()
session_info['time_start'] = time.strftime(
'%FT%T%z',
time.localtime(time_start),
)
runner.runall(testcases, restored_cases)
finally:
time_end = time.time()
session_info['time_end'] = time.strftime('%FT%T%z',
time.localtime(time_end))
session_info['time_elapsed'] = time_end - time_start
# Print a retry report if we did any retries
if runner.stats.failed(run=0):
printer.info(runner.stats.retry_report())
# Print a failure report if we had failures in the last run
success = True
if runner.stats.failed():
success = False
runner.stats.print_failure_report(printer)
if options.failure_stats:
runner.stats.print_failure_stats(printer)
if options.performance_report:
printer.info(runner.stats.performance_report())
# Generate the report for this session
report_file = os.path.normpath(
osext.expandvars(rt.get_option('general/0/report_file')))
basedir = os.path.dirname(report_file)
if basedir:
os.makedirs(basedir, exist_ok=True)
# Build final JSON report
run_stats = runner.stats.json()
session_info.update({
'num_cases': run_stats[0]['num_cases'],
'num_failures': run_stats[-1]['num_failures']
})
json_report = {
'session_info': session_info,
'runs': run_stats,
'restored_cases': []
}
if options.restore_session is not None:
for c in restored_cases:
json_report['restored_cases'].append(report.case(*c))
report_file = runreport.next_report_filename(report_file)
try:
with open(report_file, 'w') as fp:
jsonext.dump(json_report, fp, indent=2)
fp.write('\n')
printer.info(f'Run report saved in {report_file!r}')
except OSError as e:
printer.warning(
f'failed to generate report in {report_file!r}: {e}')
# Generate the junit xml report for this session
junit_report_file = rt.get_option('general/0/report_junit')
if junit_report_file:
# Expand variables in filename
junit_report_file = osext.expandvars(junit_report_file)
junit_xml = runreport.junit_xml_report(json_report)
try:
with open(junit_report_file, 'w') as fp:
runreport.junit_dump(junit_xml, fp)
except OSError as e:
printer.warning(
f'failed to generate report in {junit_report_file!r}: '
f'{e}')
if not success:
sys.exit(1)
sys.exit(0)
except (Exception, KeyboardInterrupt, errors.ReframeFatalError):
exc_info = sys.exc_info()
tb = ''.join(traceback.format_exception(*exc_info))
printer.error(f'run session stopped: {errors.what(*exc_info)}')
if errors.is_exit_request(*exc_info):
# Print stack traces for exit requests only when TOO verbose
printer.debug2(tb)
elif errors.is_severe(*exc_info):
printer.error(tb)
else:
printer.verbose(tb)
sys.exit(1)
finally:
try:
log_files = logging.log_files()
if site_config.get('general/0/save_log_files'):
log_files = logging.save_log_files(rt.output_prefix)
except OSError as e:
printer.error(f'could not save log file: {e}')
sys.exit(1)
finally:
printer.info(logfiles_message())
def partition_by_name(name):
for p in rt.runtime().system.partitions:
if p.name == name:
return p
return None
def setup_local_execution(self):
self.partition = rt.runtime().system.partition('login')
self.progenv = self.partition.environment('builtin-gcc')
def temp_prefix(self):
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def __init__(self):
self._prefix = '#BSUB'
self._submit_timeout = rt.runtime().get_option(
f'schedulers/@{self.registered_name}/job_submit_timeout'
)
def test_global_config(basic_config):
rlog.configure_logging(rt.runtime().site_config)
assert rlog.getlogger() is not rlog.null_logger
def check_sanity(self):
return runtime().modules_system.is_module_loaded('PrgEnv-cray')
def test_handler_level(basic_config, logfile):
rlog.configure_logging(rt.runtime().site_config)
rlog.getlogger().info('foo')
rlog.getlogger().warning('bar')
assert not _found_in_logfile('foo', logfile)
assert _found_in_logfile('bar', logfile)
def test_date_format(basic_config, logfile):
rlog.configure_logging(rt.runtime().site_config)
rlog.getlogger().warning('foo')
assert _found_in_logfile(datetime.now().strftime('%F'), logfile)
def test_valid_level(basic_config):
rlog.configure_logging(rt.runtime().site_config)
assert rlog.INFO == rlog.getlogger().getEffectiveLevel()
def test_autotect_with_invalid_files(invalid_topo_exec_ctx):
autodetect.detect_topology()
part = runtime().system.partitions[0]
assert part.processor.info == cpuinfo()
assert part.devices == []
def test_partition(self):
p = rt.runtime().system.partition('gpu')
assert 2 == self.count_checks(filters.have_partition([p]))
p = rt.runtime().system.partition('login')
assert 0 == self.count_checks(filters.have_partition([p]))
def setUp(self):
self.partition = rt.runtime().system.partition('login')
self.prgenv = self.partition.environment('builtin-gcc')
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def user_runtime():
if fixtures.USER_CONFIG_FILE:
with rt.temp_runtime(fixtures.USER_CONFIG_FILE, fixtures.USER_SYSTEM):
yield rt.runtime()
else:
yield rt.runtime()
def tearDown(self):
os_ext.rmtree(rt.runtime().resources.prefix)
def assert_modules_loaded(modules):
modsys = rt.runtime().modules_system
for m in modules:
assert modsys.is_module_loaded(m)
def test_sarus(self):
partition, environ = _setup_remote_execution()
self._skip_if_not_configured(partition, 'Sarus')
with tempfile.TemporaryDirectory(dir='unittests') as dirname:
rt.runtime().resources.prefix = dirname
_run(self.create_test('Sarus', 'ubuntu:18.04'), partition, environ)
def find_modules(substr, environ_mapping=None):
'''Return all modules in the current system that contain ``substr`` in
their name.
This function is a generator and will yield tuples of partition,
environment and module combinations for each partition of the current
system and for each environment of a partition.
The ``environ_mapping`` argument allows you to map module name patterns to
ReFrame environments. This is useful for flat module name schemes, in
order to avoid incompatible combinations of modules and environments.
You can use this function to parametrize regression tests over the
available environment modules. The following example will generate tests
for all the available ``netcdf`` packages in the system:
.. code:: python
@rfm.simple_test
class MyTest(rfm.RegressionTest):
module_info = parameter(find_modules('netcdf'))
@rfm.run_after('init')
def apply_module_info(self):
s, e, m = self.module_info
self.valid_systems = [s]
self.valid_prog_environs = [e]
self.modules = [m]
...
The following example shows the use of ``environ_mapping`` with flat
module name schemes. In this example, the toolchain for which the package
was built is encoded in the module's name. Using the ``environ_mapping``
argument we can map module name patterns to ReFrame environments, so that
invalid combinations are pruned:
.. code:: python
my_find_modules = functools.partial(find_modules, environ_mapping={
r'.*CrayGNU.*': 'PrgEnv-gnu',
r'.*CrayIntel.*': 'PrgEnv-intel',
r'.*CrayCCE.*': 'PrgEnv-cray'
})
@rfm.simple_test
class MyTest(rfm.RegressionTest):
module_info = parameter(my_find_modules('GROMACS'))
@rfm.run_after('init')
def apply_module_info(self):
s, e, m = self.module_info
self.valid_systems = [s]
self.valid_prog_environs = [e]
self.modules = [m]
...
:arg substr: A substring that the returned module names must contain.
:arg environ_mapping: A dictionary mapping regular expressions to
environment names.
:returns: An iterator that iterates over tuples of the module, partition
and environment name combinations that were found.
'''
import reframe.core.runtime as rt
if not isinstance(substr, str):
raise TypeError("'substr' argument must be a string")
if (environ_mapping is not None and
not isinstance(environ_mapping, typ.Dict[str, str])):
raise TypeError(
"'environ_mapping' argument must be of type Dict[str,str]"
)
def _is_valid_for_env(m, e):
if environ_mapping is None:
return True
for patt, env in environ_mapping.items():
if re.match(patt, m) and e == env:
return True
return False
ms = rt.runtime().modules_system
current_system = rt.runtime().system
snap0 = rt.snapshot()
for p in current_system.partitions:
for e in p.environs:
rt.loadenv(p.local_env, e)
modules = ms.available_modules(substr)
snap0.restore()
for m in modules:
if _is_valid_for_env(m, e.name):
yield (p.fullname, e.name, m)
def add_task(self, task):
current_run = rt.runtime().current_run
if current_run == len(self._tasks):
self._tasks.append([])
self._tasks[current_run].append(task)
def __init__(self, name='env_snapshot'):
super().__init__(name,
runtime().modules_system.loaded_modules(),
os.environ.items())
def has_sane_modules_system():
return not isinstance(rt.runtime().modules_system.backend,
(modules.NoModImpl, modules.SpackImpl))
def _temp_runtime(config_file, system=None, options={}):
options.update({'systems/prefix': str(tmp_path)})
with rt.temp_runtime(config_file, system, options):
yield rt.runtime()
def setUp(self):
self.setup_local_execution()
self.loader = RegressionCheckLoader(['unittests/resources/checks'])
# Set runtime prefix
rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
def set_max_jobs(self, value):
for p in rt.runtime().system.partitions:
p._max_jobs = value
def main():
# Setup command line options
argparser = argparse.ArgumentParser()
output_options = argparser.add_argument_group('Options controlling output')
locate_options = argparser.add_argument_group(
'Options for locating checks')
select_options = argparser.add_argument_group(
'Options for selecting checks')
action_options = argparser.add_argument_group(
'Options controlling actions')
run_options = argparser.add_argument_group(
'Options controlling execution of checks')
env_options = argparser.add_argument_group(
'Options controlling environment')
misc_options = argparser.add_argument_group('Miscellaneous options')
# Output directory options
output_options.add_argument('--prefix',
action='store',
metavar='DIR',
help='Set output directory prefix to DIR')
output_options.add_argument('-o',
'--output',
action='store',
metavar='DIR',
help='Set output directory to DIR')
output_options.add_argument('-s',
'--stage',
action='store',
metavar='DIR',
help='Set stage directory to DIR')
output_options.add_argument(
'--perflogdir',
action='store',
metavar='DIR',
help='Set directory prefix for the performance logs '
'(default: ${prefix}/perflogs, '
'relevant only if the filelog backend is used)')
output_options.add_argument(
'--keep-stage-files',
action='store_true',
help='Keep stage directory even if check is successful')
output_options.add_argument(
'--save-log-files',
action='store_true',
default=False,
help='Copy the log file from the work dir to the output dir at the '
'end of the program')
# Check discovery options
locate_options.add_argument(
'-c',
'--checkpath',
action='store',
metavar='DIR|FILE',
help="Search for checks in DIR or FILE; multiple paths can be "
"separated with `:'")
locate_options.add_argument('-R',
'--recursive',
action='store_true',
help='Load checks recursively')
locate_options.add_argument('--ignore-check-conflicts',
action='store_true',
help='Skip checks with conflicting names')
# Select options
select_options.add_argument('-t',
'--tag',
action='append',
dest='tags',
default=[],
help='Select checks matching TAG')
select_options.add_argument('-n',
'--name',
action='append',
dest='names',
default=[],
metavar='NAME',
help='Select checks with NAME')
select_options.add_argument('-x',
'--exclude',
action='append',
dest='exclude_names',
metavar='NAME',
default=[],
help='Exclude checks with NAME')
select_options.add_argument(
'-p',
'--prgenv',
action='append',
default=[r'.*'],
help='Select tests for PRGENV programming environment only')
select_options.add_argument('--gpu-only',
action='store_true',
help='Select only GPU tests')
select_options.add_argument('--cpu-only',
action='store_true',
help='Select only CPU tests')
# Action options
action_options.add_argument('-l',
'--list',
action='store_true',
help='List matched regression checks')
action_options.add_argument(
'-L',
'--list-detailed',
action='store_true',
help='List matched regression checks with a detailed description')
action_options.add_argument('-r',
'--run',
action='store_true',
help='Run regression with the selected checks')
# Run options
run_options.add_argument('-A',
'--account',
action='store',
help='Use ACCOUNT for submitting jobs')
run_options.add_argument('-P',
'--partition',
action='store',
metavar='PART',
help='Use PART for submitting jobs')
run_options.add_argument('--reservation',
action='store',
metavar='RES',
help='Use RES for submitting jobs')
run_options.add_argument('--nodelist',
action='store',
help='Run checks on the selected list of nodes')
run_options.add_argument(
'--exclude-nodes',
action='store',
metavar='NODELIST',
help='Exclude the list of nodes from running checks')
run_options.add_argument('--job-option',
action='append',
metavar='OPT',
dest='job_options',
default=[],
help='Pass OPT to job scheduler')
run_options.add_argument('--force-local',
action='store_true',
help='Force local execution of checks')
run_options.add_argument('--skip-sanity-check',
action='store_true',
help='Skip sanity checking')
run_options.add_argument('--skip-performance-check',
action='store_true',
help='Skip performance checking')
run_options.add_argument('--strict',
action='store_true',
help='Force strict performance checking')
run_options.add_argument('--skip-system-check',
action='store_true',
help='Skip system check')
run_options.add_argument('--skip-prgenv-check',
action='store_true',
help='Skip prog. environment check')
run_options.add_argument(
'--exec-policy',
metavar='POLICY',
action='store',
choices=['async', 'serial'],
default='async',
help='Specify the execution policy for running the regression tests. '
'Available policies: "async" (default), "serial"')
run_options.add_argument('--mode',
action='store',
help='Execution mode to use')
run_options.add_argument(
'--max-retries',
metavar='NUM',
action='store',
default=0,
help='Specify the maximum number of times a failed regression test '
'may be retried (default: 0)')
run_options.add_argument(
'--flex-alloc-tasks',
action='store',
dest='flex_alloc_tasks',
metavar='{all|idle|NUM}',
default=None,
help='*deprecated*, please use --flex-alloc-nodes instead')
run_options.add_argument(
'--flex-alloc-nodes',
action='store',
dest='flex_alloc_nodes',
metavar='{all|idle|NUM}',
default=None,
help="Strategy for flexible node allocation (default: 'idle').")
env_options.add_argument('-M',
'--map-module',
action='append',
metavar='MAPPING',
dest='module_mappings',
default=[],
help='Apply a single module mapping')
env_options.add_argument(
'-m',
'--module',
action='append',
default=[],
metavar='MOD',
dest='user_modules',
help='Load module MOD before running the regression suite')
env_options.add_argument('--module-mappings',
action='store',
metavar='FILE',
dest='module_map_file',
help='Apply module mappings defined in FILE')
env_options.add_argument(
'-u',
'--unload-module',
action='append',
metavar='MOD',
dest='unload_modules',
default=[],
help='Unload module MOD before running the regression suite')
env_options.add_argument(
'--purge-env',
action='store_true',
dest='purge_env',
default=False,
help='Purge environment before running the regression suite')
# Miscellaneous options
misc_options.add_argument(
'-C',
'--config-file',
action='store',
dest='config_file',
metavar='FILE',
default=os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py'),
help='Specify a custom config-file for the machine. '
'(default: %s' %
os.path.join(reframe.INSTALL_PREFIX, 'reframe/settings.py'))
misc_options.add_argument('--nocolor',
action='store_false',
dest='colorize',
default=True,
help='Disable coloring of output')
misc_options.add_argument('--performance-report',
action='store_true',
help='Print the performance report')
# FIXME: This should move to env_options as soon as
# https://github.com/eth-cscs/reframe/pull/946 is merged
misc_options.add_argument('--non-default-craype',
action='store_true',
default=False,
help='Test a non-default Cray PE')
misc_options.add_argument(
'--show-config',
action='store_true',
help='Print configuration of the current system and exit')
misc_options.add_argument(
'--show-config-env',
action='store',
metavar='ENV',
help='Print configuration of environment ENV and exit')
misc_options.add_argument('--system',
action='store',
help='Load SYSTEM configuration explicitly')
misc_options.add_argument(
'--timestamp',
action='store',
nargs='?',
const='%FT%T',
metavar='TIMEFMT',
help='Append a timestamp component to the regression directories'
'(default format "%%FT%%T")')
misc_options.add_argument('-V',
'--version',
action='version',
version=os_ext.reframe_version())
misc_options.add_argument('-v',
'--verbose',
action='count',
default=0,
help='Increase verbosity level of output')
if len(sys.argv) == 1:
argparser.print_help()
sys.exit(1)
# Parse command line
options = argparser.parse_args()
# Load configuration
try:
settings = config.load_settings_from_file(options.config_file)
except (OSError, ReframeError) as e:
sys.stderr.write('%s: could not load settings: %s\n' %
(sys.argv[0], e))
sys.exit(1)
# Configure logging
try:
logging.configure_logging(settings.logging_config),
except (OSError, ConfigError) as e:
sys.stderr.write('could not configure logging: %s\n' % e)
sys.exit(1)
# Set colors in logger
logging.getlogger().colorize = options.colorize
# Setup printer
printer = PrettyPrinter()
printer.colorize = options.colorize
if options.verbose:
printer.inc_verbosity(options.verbose)
try:
runtime.init_runtime(settings.site_configuration,
options.system,
non_default_craype=options.non_default_craype)
except SystemAutodetectionError:
printer.warning(
'could not find a configuration entry for the current system; '
'falling back to a generic system configuration; '
'please check the online documentation on how to configure '
'ReFrame for your system.')
settings.site_configuration['systems'] = {
'generic': {
'descr': 'Generic fallback system configuration',
'hostnames': ['localhost'],
'partitions': {
'login': {
'scheduler': 'local',
'environs': ['builtin-gcc'],
'descr': 'Login nodes'
}
}
}
}
settings.site_configuration['environments'] = {
'*': {
'builtin-gcc': {
'type': 'ProgEnvironment',
'cc': 'gcc',
'cxx': 'g++',
'ftn': 'gfortran',
}
}
}
runtime.init_runtime(settings.site_configuration,
'generic',
non_default_craype=options.non_default_craype)
except Exception as e:
printer.error('configuration error: %s' % e)
printer.verbose(''.join(traceback.format_exception(*sys.exc_info())))
sys.exit(1)
rt = runtime.runtime()
try:
if options.module_map_file:
rt.modules_system.load_mapping_from_file(options.module_map_file)
if options.module_mappings:
for m in options.module_mappings:
rt.modules_system.load_mapping(m)
except (ConfigError, OSError) as e:
printer.error('could not load module mappings: %s' % e)
sys.exit(1)
if options.mode:
try:
mode_args = rt.mode(options.mode)
# Parse the mode's options and reparse the command-line
options = argparser.parse_args(mode_args)
options = argparser.parse_args(namespace=options)
except ConfigError as e:
printer.error('could not obtain execution mode: %s' % e)
sys.exit(1)
# Adjust system directories
if options.prefix:
# if prefix is set, reset all other directories
rt.resources.prefix = os_ext.expandvars(options.prefix)
rt.resources.outputdir = None
rt.resources.stagedir = None
if options.output:
rt.resources.outputdir = os_ext.expandvars(options.output)
if options.stage:
rt.resources.stagedir = os_ext.expandvars(options.stage)
if (os_ext.samefile(rt.resources.stage_prefix, rt.resources.output_prefix)
and not options.keep_stage_files):
printer.error('stage and output refer to the same directory; '
'if this is on purpose, please use also the '
"`--keep-stage-files' option.")
sys.exit(1)
if options.timestamp:
rt.resources.timefmt = options.timestamp
# Configure performance logging
# NOTE: we need resources to be configured in order to set the global
# perf. logging prefix correctly
if options.perflogdir:
rt.resources.perflogdir = os_ext.expandvars(options.perflogdir)
logging.LOG_CONFIG_OPTS['handlers.filelog.prefix'] = (
rt.resources.perflog_prefix)
# Show configuration after everything is set up
if options.show_config:
printer.info(rt.show_config())
sys.exit(0)
if options.show_config_env:
envname = options.show_config_env
for p in rt.system.partitions:
environ = p.environment(envname)
if environ:
break
if environ is None:
printer.error('no such environment: ' + envname)
sys.exit(1)
printer.info(environ.details())
sys.exit(0)
if hasattr(settings, 'perf_logging_config'):
try:
logging.configure_perflogging(settings.perf_logging_config)
except (OSError, ConfigError) as e:
printer.error('could not configure performance logging: %s\n' % e)
sys.exit(1)
else:
printer.warning('no performance logging is configured; '
'please check documentation')
# Setup the check loader
if options.checkpath:
load_path = []
for d in options.checkpath.split(':'):
d = os_ext.expandvars(d)
if not os.path.exists(d):
printer.warning("%s: path `%s' does not exist. Skipping..." %
(argparser.prog, d))
continue
load_path.append(os.path.realpath(d))
load_path = os_ext.unique_abs_paths(load_path,
prune_children=options.recursive)
loader = RegressionCheckLoader(
load_path,
recurse=options.recursive,
ignore_conflicts=options.ignore_check_conflicts)
else:
loader = RegressionCheckLoader(load_path=settings.checks_path,
prefix=reframe.INSTALL_PREFIX,
recurse=settings.checks_path_recurse)
printer.debug(argparse.format_options(options))
# Print command line
printer.info('Command line: %s' % ' '.join(sys.argv))
printer.info('Reframe version: ' + reframe.VERSION)
printer.info('Launched by user: '******'<unknown>'))
printer.info('Launched on host: ' + socket.gethostname())
# Print important paths
printer.info('Reframe paths')
printer.info('=============')
printer.info(' Check prefix : %s' % loader.prefix)
printer.info(
'%03s Check search path : %s' %
('(R)' if loader.recurse else '', "'%s'" % ':'.join(loader.load_path)))
printer.info(' Stage dir prefix : %s' % rt.resources.stage_prefix)
printer.info(' Output dir prefix : %s' % rt.resources.output_prefix)
printer.info(
' Perf. logging prefix : %s' %
os.path.abspath(logging.LOG_CONFIG_OPTS['handlers.filelog.prefix']))
try:
# Locate and load checks
try:
checks_found = loader.load_all()
except OSError as e:
raise ReframeError from e
# Filter checks by name
checks_matched = checks_found
if options.exclude_names:
for name in options.exclude_names:
checks_matched = filter(filters.have_not_name(name),
checks_matched)
if options.names:
checks_matched = filter(filters.have_name('|'.join(options.names)),
checks_matched)
# Filter checks by tags
for tag in options.tags:
checks_matched = filter(filters.have_tag(tag), checks_matched)
# Filter checks by prgenv
if not options.skip_prgenv_check:
for prgenv in options.prgenv:
checks_matched = filter(filters.have_prgenv(prgenv),
checks_matched)
# Filter checks by system
if not options.skip_system_check:
checks_matched = filter(
filters.have_partition(rt.system.partitions), checks_matched)
# Filter checks further
if options.gpu_only and options.cpu_only:
printer.error("options `--gpu-only' and `--cpu-only' "
"are mutually exclusive")
sys.exit(1)
if options.gpu_only:
checks_matched = filter(filters.have_gpu_only(), checks_matched)
elif options.cpu_only:
checks_matched = filter(filters.have_cpu_only(), checks_matched)
# Determine the allowed programming environments
allowed_environs = {
e.name
for env_patt in options.prgenv for p in rt.system.partitions
for e in p.environs if re.match(env_patt, e.name)
}
# Generate the test cases, validate dependencies and sort them
checks_matched = list(checks_matched)
testcases = generate_testcases(checks_matched,
options.skip_system_check,
options.skip_prgenv_check,
allowed_environs)
testgraph = dependency.build_deps(testcases)
dependency.validate_deps(testgraph)
testcases = dependency.toposort(testgraph)
# Unload regression's module and load user-specified modules
if hasattr(settings, 'reframe_module'):
printer.warning(
"the 'reframe_module' configuration option will be ignored; "
"please use the '-u' or '--unload-module' options")
if options.purge_env:
rt.modules_system.unload_all()
else:
for m in options.unload_modules:
rt.modules_system.unload_module(m)
# Load the environment for the current system
try:
env.load(rt.system.preload_environ)
except EnvironError as e:
printer.error("failed to load current system's environment; "
"please check your configuration")
printer.debug(str(e))
raise
for m in options.user_modules:
try:
rt.modules_system.load_module(m, force=True)
except EnvironError as e:
printer.warning("could not load module '%s' correctly: "
"Skipping..." % m)
printer.debug(str(e))
if options.flex_alloc_tasks:
printer.warning("`--flex-alloc-tasks' is deprecated and "
"will be removed in the future; "
"you should use --flex-alloc-nodes instead")
options.flex_alloc_nodes = (options.flex_alloc_nodes
or options.flex_alloc_tasks)
options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle'
# Act on checks
success = True
if options.list:
# List matched checks
list_checks(list(checks_matched), printer)
elif options.list_detailed:
# List matched checks with details
list_checks(list(checks_matched), printer, detailed=True)
elif options.run:
# Setup the execution policy
if options.exec_policy == 'serial':
exec_policy = SerialExecutionPolicy()
elif options.exec_policy == 'async':
exec_policy = AsynchronousExecutionPolicy()
else:
# This should not happen, since choices are handled by
# argparser
printer.error("unknown execution policy `%s': Exiting...")
sys.exit(1)
exec_policy.skip_system_check = options.skip_system_check
exec_policy.force_local = options.force_local
exec_policy.strict_check = options.strict
exec_policy.skip_sanity_check = options.skip_sanity_check
exec_policy.skip_performance_check = options.skip_performance_check
exec_policy.keep_stage_files = options.keep_stage_files
try:
errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
if sched_flex_alloc_nodes <= 0:
raise ConfigError(errmsg.format(options.flex_alloc_nodes))
except ValueError:
if not options.flex_alloc_nodes.casefold() in {'idle', 'all'}:
raise ConfigError(errmsg.format(
options.flex_alloc_nodes)) from None
sched_flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes
exec_policy.flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_account = options.account
exec_policy.sched_partition = options.partition
exec_policy.sched_reservation = options.reservation
exec_policy.sched_nodelist = options.nodelist
exec_policy.sched_exclude_nodelist = options.exclude_nodes
exec_policy.sched_options = options.job_options
try:
max_retries = int(options.max_retries)
except ValueError:
raise ConfigError('--max-retries is not a valid integer: %s' %
max_retries) from None
runner = Runner(exec_policy, printer, max_retries)
try:
runner.runall(testcases)
finally:
# Print a retry report if we did any retries
if runner.stats.failures(run=0):
printer.info(runner.stats.retry_report())
# Print a failure report if we had failures in the last run
if runner.stats.failures():
printer.info(runner.stats.failure_report())
success = False
if options.performance_report:
printer.info(runner.stats.performance_report())
else:
printer.error("No action specified. Please specify `-l'/`-L' for "
"listing or `-r' for running. "
"Try `%s -h' for more options." % argparser.prog)
sys.exit(1)
if not success:
sys.exit(1)
sys.exit(0)
except KeyboardInterrupt:
sys.exit(1)
except ReframeError as e:
printer.error(str(e))
sys.exit(1)
except (Exception, ReframeFatalError):
printer.error(format_exception(*sys.exc_info()))
sys.exit(1)
finally:
try:
if options.save_log_files:
logging.save_log_files(rt.resources.output_prefix)
except OSError as e:
printer.error('could not save log file: %s' % e)
sys.exit(1)
def _setup_local_execution():
partition = rt.runtime().system.partition('login')
environ = partition.environment('builtin-gcc')
return partition, environ
def main():
# Setup command line options
argparser = argparse.ArgumentParser()
output_options = argparser.add_argument_group(
'Options controlling ReFrame output'
)
locate_options = argparser.add_argument_group(
'Options for discovering checks'
)
select_options = argparser.add_argument_group(
'Options for selecting checks'
)
action_options = argparser.add_argument_group(
'Options controlling actions'
)
run_options = argparser.add_argument_group(
'Options controlling the execution of checks'
)
env_options = argparser.add_argument_group(
'Options controlling the ReFrame environment'
)
misc_options = argparser.add_argument_group('Miscellaneous options')
# Output directory options
output_options.add_argument(
'--prefix', action='store', metavar='DIR',
help='Set general directory prefix to DIR',
envvar='RFM_PREFIX', configvar='systems/prefix'
)
output_options.add_argument(
'-o', '--output', action='store', metavar='DIR',
help='Set output directory prefix to DIR',
envvar='RFM_OUTPUT_DIR', configvar='systems/outputdir'
)
output_options.add_argument(
'-s', '--stage', action='store', metavar='DIR',
help='Set stage directory prefix to DIR',
envvar='RFM_STAGE_DIR', configvar='systems/stagedir'
)
output_options.add_argument(
'--timestamp', action='store', nargs='?', const='', metavar='TIMEFMT',
help=('Append a timestamp to the output and stage directory prefixes '
'(default: "%%FT%%T")'),
envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs'
)
output_options.add_argument(
'--perflogdir', action='store', metavar='DIR',
help=('Set performance log data directory prefix '
'(relevant only to the filelog log handler)'),
envvar='RFM_PERFLOG_DIR',
configvar='logging/handlers_perflog/filelog_basedir'
)
output_options.add_argument(
'--keep-stage-files', action='store_true',
help='Keep stage directories even for successful checks',
envvar='RFM_KEEP_STAGE_FILES', configvar='general/keep_stage_files'
)
output_options.add_argument(
'--dont-restage', action='store_false', dest='clean_stagedir',
help='Reuse the test stage directory',
envvar='RFM_CLEAN_STAGEDIR', configvar='general/clean_stagedir'
)
output_options.add_argument(
'--save-log-files', action='store_true', default=False,
help='Save ReFrame log files to the output directory',
envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files'
)
output_options.add_argument(
'--report-file', action='store', metavar='FILE',
help="Store JSON run report in FILE",
envvar='RFM_REPORT_FILE',
configvar='general/report_file'
)
# Check discovery options
locate_options.add_argument(
'-c', '--checkpath', action='append', metavar='PATH',
help="Add PATH to the check search path list",
envvar='RFM_CHECK_SEARCH_PATH :', configvar='general/check_search_path'
)
locate_options.add_argument(
'-R', '--recursive', action='store_true',
help='Search for checks in the search path recursively',
envvar='RFM_CHECK_SEARCH_RECURSIVE',
configvar='general/check_search_recursive'
)
locate_options.add_argument(
'--ignore-check-conflicts', action='store_true',
help='Skip checks with conflicting names',
envvar='RFM_IGNORE_CHECK_CONFLICTS',
configvar='general/ignore_check_conflicts'
)
# Select options
select_options.add_argument(
'-t', '--tag', action='append', dest='tags', metavar='PATTERN',
default=[],
help='Select checks with at least one tag matching PATTERN'
)
select_options.add_argument(
'-n', '--name', action='append', dest='names', default=[],
metavar='PATTERN', help='Select checks whose name matches PATTERN'
)
select_options.add_argument(
'-x', '--exclude', action='append', dest='exclude_names',
metavar='PATTERN', default=[],
help='Exclude checks whose name matches PATTERN'
)
select_options.add_argument(
'-p', '--prgenv', action='append', default=[r'.*'], metavar='PATTERN',
help=('Select checks with at least one '
'programming environment matching PATTERN')
)
select_options.add_argument(
'--gpu-only', action='store_true',
help='Select only GPU checks'
)
select_options.add_argument(
'--cpu-only', action='store_true',
help='Select only CPU checks'
)
# Action options
action_options.add_argument(
'-l', '--list', action='store_true',
help='List the selected checks'
)
action_options.add_argument(
'-L', '--list-detailed', action='store_true',
help='List the selected checks providing details for each test'
)
action_options.add_argument(
'-r', '--run', action='store_true',
help='Run the selected checks'
)
# Run options
run_options.add_argument(
'-A', '--account', action='store',
help="Use ACCOUNT for submitting jobs (Slurm) "
"*deprecated*, please use '-J account=ACCOUNT'")
run_options.add_argument(
'-P', '--partition', action='store', metavar='PART',
help="Use PART for submitting jobs (Slurm/PBS/Torque) "
"*deprecated*, please use '-J partition=PART' "
"or '-J q=PART'")
run_options.add_argument(
'--reservation', action='store', metavar='RES',
help="Use RES for submitting jobs (Slurm) "
"*deprecated*, please use '-J reservation=RES'")
run_options.add_argument(
'--nodelist', action='store',
help="Run checks on the selected list of nodes (Slurm) "
"*deprecated*, please use '-J nodelist=NODELIST'")
run_options.add_argument(
'--exclude-nodes', action='store', metavar='NODELIST',
help="Exclude the list of nodes from running checks (Slurm) "
"*deprecated*, please use '-J exclude=NODELIST'")
run_options.add_argument(
'-J', '--job-option', action='append', metavar='OPT',
dest='job_options', default=[],
help='Pass option OPT to job scheduler'
)
run_options.add_argument(
'--force-local', action='store_true',
help='Force local execution of checks'
)
run_options.add_argument(
'--skip-sanity-check', action='store_true',
help='Skip sanity checking'
)
run_options.add_argument(
'--skip-performance-check', action='store_true',
help='Skip performance checking'
)
run_options.add_argument(
'--strict', action='store_true',
help='Enforce strict performance checking'
)
run_options.add_argument(
'--skip-system-check', action='store_true',
help='Skip system check'
)
run_options.add_argument(
'--skip-prgenv-check', action='store_true',
help='Skip programming environment check'
)
run_options.add_argument(
'--exec-policy', metavar='POLICY', action='store',
choices=['async', 'serial'], default='async',
help='Set the execution policy of ReFrame (default: "async")'
)
run_options.add_argument(
'--mode', action='store', help='Execution mode to use'
)
run_options.add_argument(
'--max-retries', metavar='NUM', action='store', default=0,
help='Set the maximum number of times a failed regression test '
'may be retried (default: 0)'
)
run_options.add_argument(
'--flex-alloc-nodes', action='store',
dest='flex_alloc_nodes', metavar='{all|STATE|NUM}', default=None,
help='Set strategy for the flexible node allocation (default: "idle").'
)
env_options.add_argument(
'-M', '--map-module', action='append', metavar='MAPPING',
dest='module_mappings', default=[],
help='Add a module mapping',
envvar='RFM_MODULE_MAPPINGS ,', configvar='general/module_mappings'
)
env_options.add_argument(
'-m', '--module', action='append', default=[],
metavar='MOD', dest='user_modules',
help='Load module MOD before running any regression check',
envvar='RFM_USER_MODULES ,', configvar='general/user_modules'
)
env_options.add_argument(
'--module-mappings', action='store', metavar='FILE',
dest='module_map_file',
help='Load module mappings from FILE',
envvar='RFM_MODULE_MAP_FILE', configvar='general/module_map_file'
)
env_options.add_argument(
'-u', '--unload-module', action='append', metavar='MOD',
dest='unload_modules', default=[],
help='Unload module MOD before running any regression check',
envvar='RFM_UNLOAD_MODULES ,', configvar='general/unload_modules'
)
env_options.add_argument(
'--purge-env', action='store_true', dest='purge_env', default=False,
help='Unload all modules before running any regression check',
envvar='RFM_PURGE_ENVIRONMENT', configvar='general/purge_environment'
)
env_options.add_argument(
'--non-default-craype', action='store_true',
help='Test a non-default Cray Programming Environment',
envvar='RFM_NON_DEFAULT_CRAYPE', configvar='general/non_default_craype'
)
# Miscellaneous options
misc_options.add_argument(
'-C', '--config-file', action='store',
dest='config_file', metavar='FILE',
help='Set configuration file',
envvar='RFM_CONFIG_FILE'
)
misc_options.add_argument(
'--nocolor', action='store_false', dest='colorize',
help='Disable coloring of output',
envvar='RFM_COLORIZE', configvar='general/colorize'
)
misc_options.add_argument(
'--failure-stats', action='store_true', help='Print failure statistics'
)
misc_options.add_argument(
'--performance-report', action='store_true',
help='Print a report for performance tests'
)
misc_options.add_argument(
'--show-config', action='store', nargs='?', const='all',
metavar='PARAM',
help='Print the value of configuration parameter PARAM and exit'
)
misc_options.add_argument(
'--system', action='store', help='Load configuration for SYSTEM',
envvar='RFM_SYSTEM'
)
misc_options.add_argument(
'--upgrade-config-file', action='store', metavar='OLD[:NEW]',
help='Upgrade old configuration file to new syntax'
)
misc_options.add_argument(
'-V', '--version', action='version', version=os_ext.reframe_version()
)
misc_options.add_argument(
'-v', '--verbose', action='count',
help='Increase verbosity level of output',
envvar='RFM_VERBOSE', configvar='general/verbose'
)
# Options not associated with command-line arguments
argparser.add_argument(
dest='graylog_server',
envvar='RFM_GRAYLOG_ADDRESS',
configvar='logging/handlers_perflog/graylog_address',
help='Graylog server address'
)
argparser.add_argument(
dest='syslog_address',
envvar='RFM_SYSLOG_ADDRESS',
configvar='logging/handlers_perflog/syslog_address',
help='Syslog server address'
)
argparser.add_argument(
dest='ignore_reqnodenotavail',
envvar='RFM_IGNORE_REQNODENOTAVAIL',
configvar='schedulers/ignore_reqnodenotavail',
action='store_true',
help='Graylog server address'
)
argparser.add_argument(
dest='use_login_shell',
envvar='RFM_USE_LOGIN_SHELL',
configvar='general/use_login_shell',
action='store_true',
help='Use a login shell for job scripts'
)
if len(sys.argv) == 1:
argparser.print_help()
sys.exit(1)
# Parse command line
options = argparser.parse_args()
# First configure logging with our generic configuration so as to be able
# to print pretty messages; logging will be reconfigured by user's
# configuration later
site_config = config.load_config(
os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py')
)
site_config.select_subconfig('generic')
options.update_config(site_config)
logging.configure_logging(site_config)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer = PrettyPrinter()
printer.colorize = site_config.get('general/0/colorize')
printer.inc_verbosity(site_config.get('general/0/verbose'))
if os.getenv('RFM_GRAYLOG_SERVER'):
printer.warning(
'RFM_GRAYLOG_SERVER environment variable is deprecated; '
'please use RFM_GRAYLOG_ADDRESS instead'
)
os.environ['RFM_GRAYLOG_ADDRESS'] = os.getenv('RFM_GRAYLOG_SERVER')
if options.upgrade_config_file is not None:
old_config, *new_config = options.upgrade_config_file.split(
':', maxsplit=1)
new_config = new_config[0] if new_config else None
try:
new_config = config.convert_old_config(old_config, new_config)
except Exception as e:
printer.error(f'could not convert file: {e}')
sys.exit(1)
printer.info(
f'Conversion successful! '
f'The converted file can be found at {new_config!r}.'
)
sys.exit(0)
# Now configure ReFrame according to the user configuration file
try:
try:
site_config = config.load_config(options.config_file)
except ReframeDeprecationWarning as e:
printer.warning(e)
converted = config.convert_old_config(options.config_file)
printer.warning(
f"configuration file has been converted "
f"to the new syntax here: '{converted}'"
)
site_config = config.load_config(converted)
site_config.validate()
site_config.select_subconfig(options.system)
for err in options.update_config(site_config):
printer.warning(str(err))
# Update options from the selected execution mode
if options.mode:
mode_args = site_config.get(f'modes/@{options.mode}/options')
# Parse the mode's options and reparse the command-line
options = argparser.parse_args(mode_args)
options = argparser.parse_args(namespace=options.cmd_options)
options.update_config(site_config)
logging.configure_logging(site_config)
except (OSError, ConfigError) as e:
printer.error(f'failed to load configuration: {e}')
sys.exit(1)
logging.getlogger().colorize = site_config.get('general/0/colorize')
printer.colorize = site_config.get('general/0/colorize')
printer.inc_verbosity(site_config.get('general/0/verbose'))
try:
runtime.init_runtime(site_config)
except ConfigError as e:
printer.error(f'failed to initialize runtime: {e}')
sys.exit(1)
rt = runtime.runtime()
try:
if site_config.get('general/0/module_map_file'):
rt.modules_system.load_mapping_from_file(
site_config.get('general/0/module_map_file')
)
if site_config.get('general/0/module_mappings'):
for m in site_config.get('general/0/module_mappings'):
rt.modules_system.load_mapping(m)
except (ConfigError, OSError) as e:
printer.error('could not load module mappings: %s' % e)
sys.exit(1)
if (os_ext.samefile(rt.stage_prefix, rt.output_prefix) and
not site_config.get('general/0/keep_stage_files')):
printer.error("stage and output refer to the same directory; "
"if this is on purpose, please use the "
"'--keep-stage-files' option.")
sys.exit(1)
# Show configuration after everything is set up
if options.show_config:
config_param = options.show_config
if config_param == 'all':
printer.info(str(rt.site_config))
else:
value = rt.get_option(config_param)
if value is None:
printer.error(
f'no such configuration parameter found: {config_param}'
)
else:
printer.info(json.dumps(value, indent=2))
sys.exit(0)
printer.debug(format_env(options.env_vars))
# Setup the check loader
loader = RegressionCheckLoader(
load_path=site_config.get('general/0/check_search_path'),
recurse=site_config.get('general/0/check_search_recursive'),
ignore_conflicts=site_config.get('general/0/ignore_check_conflicts')
)
def print_infoline(param, value):
param = param + ':'
printer.info(f" {param.ljust(18)} {value}")
session_info = {
'cmdline': ' '.join(sys.argv),
'config_file': rt.site_config.filename,
'data_version': '1.0',
'hostname': socket.gethostname(),
'prefix_output': rt.output_prefix,
'prefix_stage': rt.stage_prefix,
'user': os_ext.osuser(),
'version': os_ext.reframe_version(),
'workdir': os.getcwd(),
}
# Print command line
printer.info(f"[ReFrame Setup]")
print_infoline('version', session_info['version'])
print_infoline('command', repr(session_info['cmdline']))
print_infoline(
f"launched by",
f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}"
)
print_infoline('working directory', repr(session_info['workdir']))
print_infoline('settings file', f"{session_info['config_file']!r}")
print_infoline('check search path',
f"{'(R) ' if loader.recurse else ''}"
f"{':'.join(loader.load_path)!r}")
print_infoline('stage directory', repr(session_info['prefix_stage']))
print_infoline('output directory', repr(session_info['prefix_output']))
printer.info('')
try:
# Locate and load checks
try:
checks_found = loader.load_all()
except OSError as e:
raise ReframeError from e
# Filter checks by name
checks_matched = checks_found
if options.exclude_names:
for name in options.exclude_names:
checks_matched = filter(filters.have_not_name(name),
checks_matched)
if options.names:
checks_matched = filter(filters.have_name('|'.join(options.names)),
checks_matched)
# Filter checks by tags
for tag in options.tags:
checks_matched = filter(filters.have_tag(tag), checks_matched)
# Filter checks by prgenv
if not options.skip_prgenv_check:
for prgenv in options.prgenv:
checks_matched = filter(filters.have_prgenv(prgenv),
checks_matched)
# Filter checks by system
if not options.skip_system_check:
checks_matched = filter(
filters.have_partition(rt.system.partitions), checks_matched)
# Filter checks further
if options.gpu_only and options.cpu_only:
printer.error("options `--gpu-only' and `--cpu-only' "
"are mutually exclusive")
sys.exit(1)
if options.gpu_only:
checks_matched = filter(filters.have_gpu_only(), checks_matched)
elif options.cpu_only:
checks_matched = filter(filters.have_cpu_only(), checks_matched)
# Determine the allowed programming environments
allowed_environs = {e.name
for env_patt in options.prgenv
for p in rt.system.partitions
for e in p.environs if re.match(env_patt, e.name)}
# Generate the test cases, validate dependencies and sort them
checks_matched = list(checks_matched)
testcases = generate_testcases(checks_matched,
options.skip_system_check,
options.skip_prgenv_check,
allowed_environs)
testgraph = dependency.build_deps(testcases)
dependency.validate_deps(testgraph)
testcases = dependency.toposort(testgraph)
# Manipulate ReFrame's environment
if site_config.get('general/0/purge_environment'):
rt.modules_system.unload_all()
else:
for m in site_config.get('general/0/unload_modules'):
rt.modules_system.unload_module(m)
# Load the environment for the current system
try:
runtime.loadenv(rt.system.preload_environ)
except EnvironError as e:
printer.error("failed to load current system's environment; "
"please check your configuration")
printer.debug(str(e))
raise
for m in site_config.get('general/0/user_modules'):
try:
rt.modules_system.load_module(m, force=True)
except EnvironError as e:
printer.warning("could not load module '%s' correctly: "
"Skipping..." % m)
printer.debug(str(e))
options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle'
if options.account:
printer.warning(f"`--account' is deprecated and "
f"will be removed in the future; you should "
f"use `-J account={options.account}'")
if options.partition:
printer.warning(f"`--partition' is deprecated and "
f"will be removed in the future; you should "
f"use `-J partition={options.partition}' "
f"or `-J q={options.partition}' depending on your "
f"scheduler")
if options.reservation:
printer.warning(f"`--reservation' is deprecated and "
f"will be removed in the future; you should "
f"use `-J reservation={options.reservation}'")
if options.nodelist:
printer.warning(f"`--nodelist' is deprecated and "
f"will be removed in the future; you should "
f"use `-J nodelist={options.nodelist}'")
if options.exclude_nodes:
printer.warning(f"`--exclude-nodes' is deprecated and "
f"will be removed in the future; you should "
f"use `-J exclude={options.exclude_nodes}'")
# Act on checks
success = True
if options.list:
# List matched checks
list_checks(list(checks_matched), printer)
elif options.list_detailed:
# List matched checks with details
list_checks(list(checks_matched), printer, detailed=True)
elif options.run:
# Setup the execution policy
if options.exec_policy == 'serial':
exec_policy = SerialExecutionPolicy()
elif options.exec_policy == 'async':
exec_policy = AsynchronousExecutionPolicy()
else:
# This should not happen, since choices are handled by
# argparser
printer.error("unknown execution policy `%s': Exiting...")
sys.exit(1)
exec_policy.skip_system_check = options.skip_system_check
exec_policy.force_local = options.force_local
exec_policy.strict_check = options.strict
exec_policy.skip_sanity_check = options.skip_sanity_check
exec_policy.skip_performance_check = options.skip_performance_check
exec_policy.keep_stage_files = site_config.get(
'general/0/keep_stage_files'
)
try:
errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
if sched_flex_alloc_nodes <= 0:
raise ConfigError(errmsg.format(options.flex_alloc_nodes))
except ValueError:
sched_flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes
exec_policy.flex_alloc_nodes = options.flex_alloc_nodes
exec_policy.sched_account = options.account
exec_policy.sched_partition = options.partition
exec_policy.sched_reservation = options.reservation
exec_policy.sched_nodelist = options.nodelist
exec_policy.sched_exclude_nodelist = options.exclude_nodes
parsed_job_options = []
for opt in options.job_options:
if opt.startswith('-') or opt.startswith('#'):
parsed_job_options.append(opt)
elif len(opt) == 1:
parsed_job_options.append(f'-{opt}')
else:
parsed_job_options.append(f'--{opt}')
exec_policy.sched_options = parsed_job_options
try:
max_retries = int(options.max_retries)
except ValueError:
raise ConfigError('--max-retries is not a valid integer: %s' %
max_retries) from None
runner = Runner(exec_policy, printer, max_retries)
try:
time_start = time.time()
session_info['time_start'] = time.strftime(
'%FT%T%z', time.localtime(time_start),
)
runner.runall(testcases)
finally:
time_end = time.time()
session_info['time_end'] = time.strftime(
'%FT%T%z', time.localtime(time_end)
)
session_info['time_elapsed'] = time_end - time_start
# Print a retry report if we did any retries
if runner.stats.failures(run=0):
printer.info(runner.stats.retry_report())
# Print a failure report if we had failures in the last run
if runner.stats.failures():
printer.info(runner.stats.failure_report())
success = False
if options.failure_stats:
printer.info(runner.stats.failure_stats())
if options.performance_report:
printer.info(runner.stats.performance_report())
# Generate the report for this session
report_file = os.path.normpath(
os_ext.expandvars(rt.get_option('general/0/report_file'))
)
basedir = os.path.dirname(report_file)
if basedir:
os.makedirs(basedir, exist_ok=True)
# Build final JSON report
run_stats = runner.stats.json()
session_info.update({
'num_cases': run_stats[0]['num_cases'],
'num_failures': run_stats[-1]['num_failures']
})
json_report = {
'session_info': session_info,
'runs': run_stats
}
report_file = generate_report_filename(report_file)
try:
with open(report_file, 'w') as fp:
json.dump(json_report, fp, indent=2)
except OSError as e:
printer.warning(
f'failed to generate report in {report_file!r}: {e}'
)
else:
printer.error("No action specified. Please specify `-l'/`-L' for "
"listing or `-r' for running. "
"Try `%s -h' for more options." %
argparser.prog)
sys.exit(1)
if not success:
sys.exit(1)
sys.exit(0)
except KeyboardInterrupt:
sys.exit(1)
except ReframeError as e:
printer.error(str(e))
sys.exit(1)
except (Exception, ReframeFatalError):
printer.error(format_exception(*sys.exc_info()))
sys.exit(1)
finally:
try:
if site_config.get('general/0/save_log_files'):
logging.save_log_files(rt.output_prefix)
except OSError as e:
printer.error('could not save log file: %s' % e)
sys.exit(1)
def _emit_gitlab_pipeline(testcases):
config = runtime.runtime().site_config
# Collect the necessary ReFrame invariants
program = 'reframe'
prefix = 'rfm-stage/${CI_COMMIT_SHORT_SHA}'
checkpath = config.get('general/0/check_search_path')
recurse = config.get('general/0/check_search_recursive')
verbosity = 'v' * config.get('general/0/verbose')
def rfm_command(testcase):
if config.filename != '<builtin>':
config_opt = f'-C {config.filename}'
else:
config_opt = ''
report_file = f'{testcase.check.unique_name}-report.json'
if testcase.level:
restore_files = ','.join(f'{t.check.unique_name}-report.json'
for t in tc.deps)
else:
restore_files = None
return ' '.join([
program, f'--prefix={prefix}', config_opt,
f'{" ".join("-c " + c for c in checkpath)}',
f'-R' if recurse else '', f'--report-file={report_file}',
f'--restore-session={restore_files}' if restore_files else '',
f'--report-junit={testcase.check.unique_name}-report.xml',
f'{"".join("-" + verbosity)}' if verbosity else '', '-n',
f"'^{testcase.check.unique_name}$'", '-r'
])
max_level = 0 # We need the maximum level to generate the stages section
json = {
'cache': {
'key': '${CI_COMMIT_REF_SLUG}',
'paths': ['rfm-stage/${CI_COMMIT_SHORT_SHA}']
},
'stages': []
}
# Name of the image used for CI. If user does not explicitly provide
# image keyword on the top of CI script, this variable does not exist
image_name = os.getenv('CI_JOB_IMAGE')
if image_name:
json['image'] = image_name
for tc in testcases:
json[f'{tc.check.unique_name}'] = {
'stage': f'rfm-stage-{tc.level}',
'script': [rfm_command(tc)],
'artifacts': {
'paths': [f'{tc.check.unique_name}-report.json']
},
'needs': [t.check.unique_name for t in tc.deps]
}
max_level = max(max_level, tc.level)
json['stages'] = [f'rfm-stage-{m}' for m in range(max_level + 1)]
return json
