def _create_from_python(cls, filename):
try:
mod = util.import_module_from_file(filename)
except ImportError as e:
# import_module_from_file() may raise an ImportError if the
# configuration file is under ReFrame's top-level directory
raise ConfigError(
f"could not load Python configuration file: '{filename}'"
) from e
if hasattr(mod, 'settings'):
# Looks like an old style config
raise ConfigError(
f"the syntax of the configuration file {filename!r} "
f"is no longer supported; please convert it using the "
f"'--upgrade-config-file' option"
)
mod = util.import_module_from_file(filename)
if not hasattr(mod, 'site_configuration'):
raise ConfigError(
f"not a valid Python configuration file: '{filename}'"
)
return _SiteConfig(mod.site_configuration, filename)
def __init__(self):
# Try to figure out if we are indeed using the TCL version
try:
completed = os_ext.run_command('modulecmd -V')
except OSError as e:
raise ConfigError('could not find a sane Tmod installation: %s' %
e) from e
version_match = re.search(r'^VERSION=(\S+)', completed.stdout,
re.MULTILINE)
tcl_version_match = re.search(r'^TCL_VERSION=(\S+)', completed.stdout,
re.MULTILINE)
if version_match is None or tcl_version_match is None:
raise ConfigError('could not find a sane Tmod installation')
self._version = version_match.group(1)
self._command = 'modulecmd python'
try:
# Try the Python bindings now
completed = os_ext.run_command(self._command)
except OSError as e:
raise ConfigError('could not get the Python bindings for Tmod: ' %
e) from e
if re.search(r'Unknown shell type', completed.stderr):
raise ConfigError(
'Python is not supported by this Tmod installation')
def validate(self):
site_config = self._pick_config()
try:
jsonschema.validate(site_config, self._schema)
except jsonschema.ValidationError as e:
raise ConfigError(f"could not validate configuration file: "
f"'{self._filename}'") from e
# Make sure that system and partition names are unique
system_names = set()
for system in self._site_config['systems']:
sysname = system['name']
if sysname in system_names:
raise ConfigError(f"system '{sysname}' already defined")
system_names.add(sysname)
partition_names = set()
for part in system['partitions']:
partname = part['name']
if partname in partition_names:
raise ConfigError(
f"partition '{partname}' already defined "
f"for system '{sysname}'")
partition_names.add(partname)
def _create_syslog_handler(handler_config):
address = handler_config.get('address', None)
if address is None:
raise ConfigError('syslog handler: no address specified')
# Check if address is in `host:port` format
try:
host, port = address.split(':', maxsplit=1)
except ValueError:
pass
else:
address = (host, port)
facility = handler_config.get('facility', 'user')
try:
facility_type = logging.handlers.SysLogHandler.facility_names[facility]
except KeyError:
raise ConfigError('syslog handler: '
'unknown facility: %s' % facility) from None
socktype = handler_config.get('socktype', 'udp')
if socktype == 'udp':
socket_type = socket.SOCK_DGRAM
elif socktype == 'tcp':
socket_type = socket.SOCK_STREAM
else:
raise ConfigError('syslog handler: unknown socket type: %s' % socktype)
return logging.handlers.SysLogHandler(address, facility_type, socket_type)
def __init__(self):
self._command = 'modulecmd python'
try:
completed = os_ext.run_command(self._command + ' -V', check=True)
except OSError as e:
raise ConfigError(
'could not find a sane TMod4 installation') from e
except SpawnedProcessError as e:
raise ConfigError(
'could not get the Python bindings for TMod4') from e
version_match = re.match(r'^Modules Release (\S+)\s+',
completed.stderr)
if not version_match:
raise ConfigError('could not retrieve the TMod4 version')
version = version_match.group(1)
try:
ver_major, ver_minor, *_ = [int(v) for v in version.split('.')]
except ValueError:
raise ConfigError(
'could not parse TMod4 version string: ' + version) from None
if (ver_major, ver_minor) < self.MIN_VERSION:
raise ConfigError(
'unsupported TMod4 version: %s (required >= %s)' %
(version, self.MIN_VERSION))
self._version = version
def _create_httpjson_handler(site_config, config_prefix):
url = site_config.get(f'{config_prefix}/url')
parsed_url = urllib.parse.urlparse(url)
if parsed_url.scheme not in {'http', 'https'}:
raise ConfigError(
"httpjson handler: invalid url scheme: use 'http' or 'https'")
if not parsed_url.hostname:
raise ConfigError('http json handler: invalid hostname')
try:
if not parsed_url.port:
raise ConfigError('http json handler: no port given')
except ValueError as e:
raise ConfigError('http json handler: invalid port') from e
# Check if the remote server is up and accepts connections; if not we will
# skip the handler
try:
with socket.create_connection((parsed_url.hostname, parsed_url.port),
timeout=1):
pass
except OSError as e:
getlogger().warning(f'httpjson: could not connect to server'
f'{parsed_url.hostname}:{parsed_url.port}: {e}')
return None
extras = site_config.get(f'{config_prefix}/extras')
return HTTPJSONHandler(url, extras)
def __init__(self):
# Try to figure out if we are indeed using LMOD
lmod_cmd = os.getenv('LMOD_CMD')
if lmod_cmd is None:
raise ConfigError('could not find a sane Lmod installation: '
'environment variable LMOD_CMD is not defined')
try:
completed = os_ext.run_command('%s --version' % lmod_cmd)
except OSError as e:
raise ConfigError(
'could not find a sane Lmod installation: %s' % e)
version_match = re.search(r'.*Version\s*(\S+)', completed.stderr,
re.MULTILINE)
if version_match is None:
raise ConfigError('could not retrieve Lmod version')
self._version = version_match.group(1)
self._command = '%s python ' % lmod_cmd
try:
# Try the Python bindings now
completed = os_ext.run_command(self._command)
except OSError as e:
raise ConfigError(
'could not get the Python bindings for Lmod: ' % e)
if re.search(r'Unknown shell type', completed.stderr):
raise ConfigError('Python is not supported by '
'this Lmod installation')
def _exec_module_command(self, *args, msg=None):
completed = self._run_module_command(*args, msg=msg)
exec_match = re.search(r'^exec\s\'', completed.stdout)
if exec_match is None:
raise ConfigError('could not use the python bindings')
else:
cmd = completed.stdout
exec_match = re.search(r'^exec\s\'(\S+)\'', cmd, re.MULTILINE)
if exec_match is None:
raise ConfigError('could not use the python bindings')
with open(exec_match.group(1), 'r') as content_file:
cmd = content_file.read()
exec(cmd)
def _extract_handlers(handlers_list):
# Check if we are using the old syntax
if isinstance(handlers_list, collections.abc.Mapping):
handlers_list = _convert_handler_syntax(handlers_list)
sys.stderr.write(
'WARNING: looks like you are using an old syntax for the '
'logging configuration; please update your syntax as follows:\n'
'\nhandlers: %s\n' % pprint.pformat(handlers_list, indent=1))
handlers = []
if not handlers_list:
raise ValueError('no handlers are defined for logging')
for i, handler_config in enumerate(handlers_list):
if not isinstance(handler_config, collections.abc.Mapping):
raise TypeError('handler config at position %s '
'is not a dictionary' % i)
try:
handler_type = handler_config['type']
except KeyError:
raise ConfigError('no type specified for '
'handler at position %s' % i) from None
if handler_type == 'file':
hdlr = _create_file_handler(handler_config)
elif handler_type == 'filelog':
hdlr = _create_filelog_handler(handler_config)
elif handler_type == 'syslog':
hdlr = _create_syslog_handler(handler_config)
elif handler_type == 'stream':
hdlr = _create_stream_handler(handler_config)
elif handler_type == 'graylog':
hdlr = _create_graylog_handler(handler_config)
if hdlr is None:
sys.stderr.write('WARNING: could not initialize the '
'graylog handler; ignoring...\n')
continue
else:
raise ConfigError('unknown handler type: %s' % handler_type)
level = handler_config.get('level', 'debug').lower()
fmt = handler_config.get('format', '%(message)s')
datefmt = handler_config.get('datefmt', '%FT%T')
hdlr.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt))
hdlr.setLevel(_check_level(level))
handlers.append(hdlr)
return handlers
def load_mapping(self, mapping):
"""Updates the internal module mapping with a single mapping"""
key, *rest = mapping.split(':')
if len(rest) != 1:
raise ConfigError('invalid mapping syntax: %s' % mapping)
key = key.strip()
values = rest[0].split()
if not key:
raise ConfigError('no key found in mapping: %s' % mapping)
if not values:
raise ConfigError('no mapping defined for module: %s' % key)
self.module_map[key] = list(OrderedDict.fromkeys(values))
def _detect_system(self):
getlogger().debug('Detecting system')
if os.path.exists('/etc/xthostname'):
# Get the cluster name on Cray systems
getlogger().debug(
"Found '/etc/xthostname': will use this to get the system name"
)
with open('/etc/xthostname') as fp:
hostname = fp.read()
else:
hostname = socket.getfqdn()
getlogger().debug(
f'Looking for a matching configuration entry '
f'for system {hostname!r}'
)
for system in self._site_config['systems']:
for patt in system['hostnames']:
if re.match(patt, hostname):
sysname = system['name']
getlogger().debug(
f'Configuration found: picking system {sysname!r}'
)
return sysname
raise ConfigError(f"could not find a configuration entry "
f"for the current system: '{hostname}'")
def _create_syslog_handler(site_config, config_prefix):
address = site_config.get(f'{config_prefix}/address')
# Check if address is in `host:port` format
try:
host, port = address.split(':', maxsplit=1)
except ValueError:
pass
else:
try:
address = (host, int(port))
except ValueError:
raise ConfigError(
f'syslog address port not an integer: {port!r}') from None
facility = site_config.get(f'{config_prefix}/facility')
try:
facility_type = logging.handlers.SysLogHandler.facility_names[facility]
except KeyError:
# This should not happen
raise AssertionError(
f'syslog handler: unknown facility: {facility}') from None
socktype = site_config.get(f'{config_prefix}/socktype')
if socktype == 'udp':
socket_type = socket.SOCK_DGRAM
elif socktype == 'tcp':
socket_type = socket.SOCK_STREAM
else:
# This should not happen
raise AssertionError(
f'syslog handler: unknown socket type: {socktype}')
return logging.handlers.SysLogHandler(address, facility_type, socket_type)
def _create_graylog_handler(site_config, config_prefix):
try:
import pygelf
except ImportError:
return None
address = site_config.get(f'{config_prefix}/address')
host, *port = address.split(':', maxsplit=1)
if not port:
raise ConfigError('graylog handler: no port specified')
port = port[0]
# Check if the remote server is up and accepts connections; if not we will
# skip the handler
try:
with socket.create_connection((host, port), timeout=1):
pass
except OSError as e:
getlogger().warning(
f"could not connect to Graylog server at '{address}': {e}")
return None
extras = site_config.get(f'{config_prefix}/extras')
return pygelf.GelfHttpHandler(host=host,
port=port,
static_fields=extras,
include_extra_fields=True,
json_default=jsonext.encode)
def set_autodetect_meth(self, method, **opts):
self._autodetect_meth = method
try:
self._autodetect_opts[method].update(opts)
except KeyError:
raise ConfigError(
f'unknown auto-detection method: {method!r}') from None
def create(cls, filename):
_, ext = os.path.splitext(filename)
if ext == '.py':
return cls._create_from_python(filename)
elif ext == '.json':
return cls._create_from_json(filename)
else:
raise ConfigError(f"unknown configuration file type: '{filename}'")
def load_settings_from_file(filename):
global _settings
try:
_settings = util.import_module_from_file(filename).settings
return _settings
except Exception as e:
raise ConfigError("could not load configuration file `%s'" %
filename) from e
def create_env(system, partition, name):
# Create an environment instance
try:
config = envconfig['%s:%s:%s' % (system, partition, name)]
except KeyError:
raise ConfigError("could not find a definition for `%s'" %
name) from None
if not isinstance(config, collections.abc.Mapping):
raise TypeError("config for `%s' is not a dictionary" % name)
try:
envtype = m_env.__dict__[config['type']]
return envtype(name, **config)
except KeyError:
raise ConfigError("no type specified for environment `%s'" %
name) from None
def _create_stream_handler(handler_config):
stream = handler_config.get('name', 'stdout')
if stream == 'stdout':
return logging.StreamHandler(stream=sys.stdout)
elif stream == 'stderr':
return logging.StreamHandler(stream=sys.stderr)
else:
raise ConfigError('unknown stream: %s' % stream)
def _get_backend(name, *, backend_type):
backend_modules = globals()[f'_{backend_type}_backend_modules']
for mod in backend_modules:
importlib.import_module(mod)
try:
return globals()[f'_{backend_type}s'][name]
except KeyError:
raise ConfigError(f"no such {backend_type}: '{name}'")
def _create_filelog_handler(handler_config):
logdir = os.path.abspath(LOG_CONFIG_OPTS['handlers.filelog.prefix'])
try:
filename_patt = os.path.join(logdir, handler_config['prefix'])
except KeyError:
raise ConfigError('no file specified for file handler:\n%s' %
pprint.pformat(handler_config)) from None
append = handler_config.get('append', False)
return MultiFileHandler(filename_patt, mode='a+' if append else 'w+')
def _create_from_json(cls, filename):
with open(filename) as fp:
try:
config = json.loads(fp.read())
except json.JSONDecodeError as e:
raise ConfigError(
f"invalid JSON syntax in configuration file '{filename}'"
) from e
return _SiteConfig(config, filename)
def __init__(self):
# Try to figure out if we are indeed using the TCL version
try:
completed = osext.run_command('spack -V')
except OSError as e:
raise ConfigError(
'could not find a sane Spack installation') from e
self._version = completed.stdout.strip()
self._name_format = '{name}/{version}-{hash}'
def _load_info(filename, schema=None):
try:
with open(filename) as fp:
return _validate_info(json.load(fp), schema)
except OSError as e:
getlogger().warning(f'could not load file: {filename!r}: {e}')
return {}
except jsonschema.ValidationError as e:
raise ConfigError(
f'could not validate meta-config file {filename!r}') from e
def do_register(cls):
registry = globals()[f'_{backend_type}s']
if name in registry:
raise ConfigError(
f"'{name}' is already registered as a {backend_type}")
cls.is_local = fields.ConstantField(bool(local))
cls.registered_name = fields.ConstantField(name)
registry[name] = cls
return cls
def create(cls, modules_kind=None):
if modules_kind is None:
return ModulesSystem(NoModImpl())
elif modules_kind == 'tmod':
return ModulesSystem(TModImpl())
elif modules_kind == 'tmod4':
return ModulesSystem(TMod4Impl())
elif modules_kind == 'lmod':
return ModulesSystem(LModImpl())
else:
raise ConfigError('unknown module system: %s' % modules_kind)
def load_mapping(self, mapping):
"""Update the internal module mappings using a single mapping.
:arg mapping: a string specifying the module mapping.
Example syntax: ``'m0: m1 m2'``.
"""
key, *rest = mapping.split(':')
if len(rest) != 1:
raise ConfigError('invalid mapping syntax: %s' % mapping)
key = key.strip()
values = rest[0].split()
if not key:
raise ConfigError('no key found in mapping: %s' % mapping)
if not values:
raise ConfigError('no mapping defined for module: %s' % key)
self.module_map[key] = list(OrderedDict.fromkeys(values))
def init_modules_system(modules_kind=None):
global _modules_system
if modules_kind is None:
_modules_system = ModulesSystem(NoModImpl())
elif modules_kind == 'tmod':
_modules_system = ModulesSystem(TModImpl())
elif modules_kind == 'lmod':
_modules_system = ModulesSystem(LModImpl())
else:
raise ConfigError('unknown module system: %s' % modules_kind)
def _create_graylog_handler(handler_config):
try:
import pygelf
except ImportError:
return None
host = handler_config.get('host', None)
port = handler_config.get('port', None)
extras = handler_config.get('extras', None)
if host is None:
raise ConfigError('graylog handler: no host specified')
if port is None:
raise ConfigError('graylog handler: no port specified')
if extras is not None and not isinstance(extras, collections.abc.Mapping):
raise ConfigError('graylog handler: extras must be a mapping type')
return pygelf.GelfHttpHandler(host=host, port=port, debug=True,
static_fields=extras,
include_extra_fields=True)
def create_env(system, partition, name):
# Create an environment instance
try:
config = envconfig['%s:%s:%s' % (system, partition, name)]
except KeyError:
raise ConfigError("could not find a definition for `%s'" %
name) from None
if not isinstance(config, collections.abc.Mapping):
raise TypeError("config for `%s' is not a dictionary" % name)
return m_env.ProgEnvironment(name, **config)
def load_mapping_from_file(self, filename):
"""Update the internal module mappings from mappings read from file."""
with open(filename) as fp:
for lineno, line in enumerate(fp, start=1):
line = line.strip().split('#')[0]
if not line:
continue
try:
self.load_mapping(line)
except ConfigError as e:
raise ConfigError('%s:%s' % (filename, lineno)) from e
