def check_metrics(self, metrics):
"""Checks compatibility of PAPI metrics.
Extracts all PAPI metrics from `metrics` and executes papi_event_chooser to check compatibility.
Args:
metrics (list): List of metrics.
Raises:
ConfigurationError: PAPI metrics are not compatible on the current host.
"""
papi_metrics = self.parse_metrics(metrics)
if not papi_metrics:
return
self.install()
event_chooser_cmd = os.path.join(self.bin_path, 'papi_event_chooser')
cmd = [event_chooser_cmd, 'PRESET'] + papi_metrics
try:
util.get_command_output(cmd)
except CalledProcessError as err:
for line in err.output.split('\n'):
if "can't be counted with others" in line:
parts = line.split()
try:
event = parts[1]
code = int(parts[-1])
except (IndexError, ValueError):
continue
if code == -1:
why = ": %s is not compatible with other events" % event
elif code == -8:
why = ": %s cannot be counted due to resource limitations" % event
else:
why = ": %s is not supported on this host" % event
break
elif "can't be found" in line:
parts = line.split()
try:
event = parts[1]
except IndexError:
continue
why = ": event %s is not available on the current host" % event
break
else:
why = ', and output from papi_event_chooser was not parsable.'
err = ConfigurationError(("PAPI metrics [%s] are not compatible on the current host%s."
"\n\nYou may ignore this warning if you are cross-compiling.") %
(', '.join(papi_metrics), why),
"Use papi_avail to check metric availability.",
"Spread the desired metrics over multiple measurements.",
"Choose fewer metrics.")
LOGGER.warning(err)
def check_metrics(self, metrics):
"""Checks compatibility of PAPI metrics.
Extracts all PAPI metrics from `metrics` and executes papi_event_chooser to check compatibility.
Args:
metrics (list): List of metrics.
Raises:
ConfigurationError: PAPI metrics are not compatible on the current host.
"""
papi_metrics = self.parse_metrics(metrics)
if not papi_metrics:
return
self.install()
event_chooser_cmd = os.path.join(self.bin_path, 'papi_event_chooser')
cmd = [event_chooser_cmd, 'PRESET'] + papi_metrics
try:
util.get_command_output(cmd)
except CalledProcessError as err:
for line in err.output.split('\n'):
if "can't be counted with others" in line:
parts = line.split()
try:
event = parts[1]
code = int(parts[-1])
except (IndexError, ValueError):
continue
if code == -1:
why = ": %s is not compatible with other events" % event
elif code == -8:
why = ": %s cannot be counted due to resource limitations" % event
else:
why = ": %s is not supported on this host" % event
break
elif "can't be found" in line:
parts = line.split()
try:
event = parts[1]
except IndexError:
continue
why = ": event %s is not available on the current host" % event
break
else:
why = ', and output from papi_event_chooser was not parsable.'
raise ConfigurationError(
("PAPI metrics [%s] are not compatible on the current host%s.")
% (', '.join(papi_metrics), why),
"Use papi_avail to check metric availability.",
"Spread the desired metrics over multiple measurements.",
"Choose fewer metrics.",
"You may ignore this if you are cross-compiling.")
def xml_event_info(self):
if not self._xml_event_info:
self.install()
xml_event_info = util.get_command_output(
os.path.join(self.bin_path, 'papi_xml_event_info'))
self._xml_event_info = ElementTree.fromstring(xml_event_info)
return self._xml_event_info
def probe(cls, absolute_path, candidates=None):
"""Determine the compiler family of a given command.
Executes the command with :any:`version_flags` from all families
and compares the output against :any:`family_regex`.
Args:
absolute_path (str): Absolute path to a compiler command.
candidates (list): If present, a list of families that are most likely.
Raises:
ConfigurationError: Compiler family could not be determined.
Returns:
_CompilerFamily: The compiler's family.
"""
try:
return cls._probe_cache[absolute_path]
except KeyError:
pass
LOGGER.debug("Probing compiler '%s' to discover compiler family", absolute_path)
messages = []
stdout = None
# Settle down pylint... the __instances__ member is created by __new__
# pylint: disable=no-member
if candidates:
candidate_kbases = {candidate.kbase for candidate in candidates}
families = candidates + [inst for inst in cls.__instances__
if inst not in candidates and inst.kbase in candidate_kbases]
else:
families = cls.__instances__
basename = os.path.basename(absolute_path)
with_regex, without_regex = [], []
for family in families:
if basename in family.commands:
(with_regex if family.family_regex else without_regex).append(family)
for family in with_regex:
cmd = [absolute_path] + family.version_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError as err:
messages.append(err.output)
LOGGER.debug("%s returned %d: %s", cmd, err.returncode, err.output)
# Keep going: Cray compilers return nonzero on version flag
if stdout:
if re.search(family.family_regex, stdout):
LOGGER.debug("'%s' is a %s compiler", absolute_path, family.name)
cls._probe_cache[absolute_path] = family
return family
else:
LOGGER.debug("'%s' is not a %s compiler", absolute_path, family.name)
if len(without_regex) > 0:
guess = without_regex[0]
if len(without_regex) > 1:
LOGGER.warning(("Assuming '%s' is a %s compiler but it could be to any of these: %s\n"
"If this assumption is incorrect then you should manually specify your compilers"),
absolute_path, guess.name, ', '.join([family.name for family in without_regex]))
return guess
raise ConfigurationError("Cannot determine compiler family: %s" % '\n'.join(messages))
def _probe_wrapper(self):
if not self.info.family.show_wrapper_flags:
return None
LOGGER.debug("Probing %s wrapper '%s'", self.info.short_descr,
self.absolute_path)
cmd = [self.absolute_path] + self.info.family.show_wrapper_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError:
# If this command didn't accept show_wrapper_flags then it's not a compiler wrapper to begin with,
# i.e. another command just happens to be the same as a known compiler command.
raise ConfigurationError(
"'%s' isn't actually a %s since it doesn't accept arguments %s."
% (self.absolute_path, self.info.short_descr,
self.info.family.show_wrapper_flags))
# Assume the longest line starting with a known compiler command is the wrapped compiler followed by arguments.
known_commands = set(info.command for info in _CompilerInfo.all())
for line in sorted(stdout.split('\n'), key=len, reverse=True):
if not line:
continue
parts = line.split()
wrapped_command = parts[0]
wrapped_args = parts[1:]
if os.path.basename(wrapped_command) not in known_commands:
continue
wrapped_absolute_path = util.which(wrapped_command)
if not wrapped_absolute_path:
continue
if wrapped_absolute_path == self.absolute_path:
# A wrapper that wraps itself isn't a wrapper, e.g. compilers that ignore invalid arguments
# when version flags are present.
return None
try:
wrapped = InstalledCompiler.probe(wrapped_command)
except ConfigurationError:
# wrapped_command might not be a real compiler command so keep trying
continue
# The wrapper must be able to perform the same role as the wrapped compiler
role = self.info.role.keyword.split('_')[1:]
wrapped_role = wrapped.info.role.keyword.split('_')[1:]
if role != wrapped_role:
raise ConfigurationError(
"Cannot use '%s' as a %s: wrapped compiler '%s' is a %s" %
(self.command, self.info.short_descr, wrapped.command,
wrapped.info.short_descr))
LOGGER.info("%s '%s' wraps '%s'", self.info.short_descr,
self.absolute_path, wrapped.absolute_path)
try:
self._parse_wrapped_args(wrapped_args)
except IndexError:
LOGGER.warning(
"Unexpected output from compiler wrapper '%s'."
" TAU will attempt to continue but may fail later on.",
self.absolute_path)
return wrapped
return None
def push_test_workdir():
"""Create a new working directory for a unit test.
Sets the current working directory and :any:`tempfile.tempdir` to the newly created test directory.
Directories created via this method are tracked. If any of them exist when the program exits then
an error message is shown for each.
"""
path = tempfile.mkdtemp()
try:
test_src = os.path.join(TAUCMDR_HOME, '.testfiles', 'foo_launcher')
test_dst = os.path.join(path, 'foo_launcher')
shutil.copy(test_src, test_dst)
get_command_output('%s/foo_launcher' % path)
except OSError:
shutil.rmtree(path)
path = tempfile.mkdtemp(dir=os.getcwd())
_DIR_STACK.append(path)
_CWD_STACK.append(os.getcwd())
_TEMPDIR_STACK.append(tempfile.tempdir)
os.chdir(path)
tempfile.tempdir = path
def _probe_wrapper(self):
if not self.info.family.show_wrapper_flags:
return None
LOGGER.debug("Probing %s wrapper '%s'", self.info.short_descr, self.absolute_path)
cmd = [self.absolute_path] + self.info.family.show_wrapper_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError:
# If this command didn't accept show_wrapper_flags then it's not a compiler wrapper to begin with,
# i.e. another command just happens to be the same as a known compiler command.
raise ConfigurationError("'%s' isn't actually a %s since it doesn't accept arguments %s." %
(self.absolute_path, self.info.short_descr, self.info.family.show_wrapper_flags))
# Assume the longest line starting with a known compiler command is the wrapped compiler followed by arguments.
known_commands = set(info.command for info in _CompilerInfo.all())
for line in sorted(stdout.split('\n'), key=len, reverse=True):
if not line:
continue
parts = line.split()
wrapped_command = parts[0]
wrapped_args = parts[1:]
if os.path.basename(wrapped_command) not in known_commands:
continue
wrapped_absolute_path = util.which(wrapped_command)
if not wrapped_absolute_path:
continue
if wrapped_absolute_path == self.absolute_path:
# A wrapper that wraps itself isn't a wrapper, e.g. compilers that ignore invalid arguments
# when version flags are present.
return None
try:
wrapped = InstalledCompiler.probe(wrapped_command)
except ConfigurationError:
# wrapped_command might not be a real compiler command so keep trying
continue
# The wrapper must be able to perform the same role as the wrapped compiler
role = self.info.role.keyword.split('_')[1:]
wrapped_role = wrapped.info.role.keyword.split('_')[1:]
if role != wrapped_role:
raise ConfigurationError("Cannot use '%s' as a %s: wrapped compiler '%s' is a %s" %
(self.command, self.info.short_descr,
wrapped.command, wrapped.info.short_descr))
LOGGER.info("%s '%s' wraps '%s'", self.info.short_descr, self.absolute_path, wrapped.absolute_path)
try:
self._parse_wrapped_args(wrapped_args)
except IndexError:
LOGGER.warning("Unexpected output from compiler wrapper '%s'."
" TAU will attempt to continue but may fail later on.", self.absolute_path)
return wrapped
return None
def _get_cmake(self):
cmake = util.which('cmake')
if not cmake:
raise ConfigurationError("'cmake' not found in PATH.")
try:
stdout = util.get_command_output([cmake, '--version'])
except (CalledProcessError, OSError) as err:
raise ConfigurationError("Failed to get CMake version: %s" % err)
for line in stdout.split('\n'):
if 'cmake version' in line:
verstr = (line.split('cmake version ')[1]).split('-')[0]
version = tuple(int(x) for x in verstr.split('.'))
if version < (2, 8):
raise ConfigurationError("CMake version 2.8 or higher required.")
break
else:
LOGGER.warning("Cannot determine CMake version. CMake 2.8 or higher is required.")
return cmake
def _get_cmake(self):
cmake = util.which('cmake')
if not cmake:
raise ConfigurationError("'cmake' not found in PATH.")
try:
stdout = util.get_command_output([cmake, '--version'])
except (CalledProcessError, OSError) as err:
raise ConfigurationError("Failed to get CMake version: %s" % err)
for line in stdout.split('\n'):
if 'cmake version' in line:
verstr = (line.split('cmake version ')[1]).split('-')[0]
version = tuple(int(x) for x in verstr.split('.'))
if version < (2, 8):
raise ConfigurationError(
"CMake version 2.8 or higher required.")
break
else:
LOGGER.warning(
"Cannot determine CMake version. CMake 2.8 or higher is required."
)
return cmake
def version_string(self):
"""Get the compiler's self-reported version info.
Usually whatever the compiler prints when the --version flag is provided.
Returns:
str: The compilers' version string.
"""
if self._version_string is None:
cmd = [self.absolute_path] + self.info.family.version_flags
try:
self._version_string = util.get_command_output(cmd)
except CalledProcessError:
raise ConfigurationError("Compiler command '%s' failed." % ' '.join(cmd),
"Check that this command works outside of TAU.",
"Check loaded modules and environment variables.",
"Verify that the compiler's license is valid.")
except OSError:
raise ConfigurationError("Compiler '%s' no longer exists or is not executable" %
self.absolute_path)
return self._version_string
def version_string(self):
"""Get the compiler's self-reported version info.
Usually whatever the compiler prints when the --version flag is provided.
Returns:
str: The compilers' version string.
"""
if self._version_string is None:
cmd = [self.absolute_path] + self.info.family.version_flags
try:
self._version_string = util.get_command_output(cmd)
except CalledProcessError:
raise ConfigurationError("Compiler command '%s' failed." % ' '.join(cmd),
"Check that this command works outside of TAU.",
"Check loaded modules and environment variables.",
"Verify that the compiler's license is valid.")
except OSError:
raise ConfigurationError("Compiler '%s' no longer exists or is not executable" %
self.absolute_path)
return self._version_string
def verify(self):
super(ScorepInstallation, self).verify()
# Use Score-P's `scorep-info` command to check if this Score-P installation
# was configured with the flags we need.
cmd = [os.path.join(self.bin_path, 'scorep-info'), 'config-summary']
try:
stdout = util.get_command_output(cmd)
except CalledProcessError as err:
raise SoftwarePackageError("%s failed with return code %d: %s" %
(cmd, err.returncode, err.output))
flags = self._get_flags()
found_flags = set()
extra_flags = set()
in_section = False
for line in stdout.splitlines():
if line.startswith('Configure command:'):
in_section = True
continue
elif in_section:
line = line.replace('./configure', '')
if not line.startswith(' '):
break
for flag in flags:
if "'%s'" % flag in line:
found_flags.add(flag)
break
else:
extra_flags.add(line.replace('\\', '').strip())
# Some extra flags are harmless
for flag in list(extra_flags):
if flag.startswith("'--prefix="):
extra_flags.remove(flag)
if found_flags != set(flags):
raise SoftwarePackageError(
"Score-P installation at '%s' was not configured with flags %s"
% (self.install_prefix, ' '.join(flags)))
if extra_flags:
raise SoftwarePackageError(
"Score-P installation at '%s' was configured with extra flags %s"
% (self.install_prefix, ' '.join(extra_flags)))
def verify(self):
super(ScorepInstallation, self).verify()
# Use Score-P's `scorep-info` command to check if this Score-P installation
# was configured with the flags we need.
cmd = [os.path.join(self.bin_path, 'scorep-info'), 'config-summary']
try:
stdout = util.get_command_output(cmd)
except CalledProcessError as err:
raise SoftwarePackageError("%s failed with return code %d: %s" % (cmd, err.returncode, err.output))
flags = self._get_flags()
found_flags = set()
extra_flags = set()
in_section = False
for line in stdout.splitlines():
if line.startswith('Configure command:'):
in_section = True
continue
elif in_section:
line = line.replace('./configure', '')
if not line.startswith(' '):
break
for flag in flags:
if ("'%s'" % flag) in line:
found_flags.add(flag)
break
else:
extra_flags.add(line.replace('\\', '').strip())
# Some extra flags are harmless
for flag in list(extra_flags):
if flag.startswith("'--prefix="):
extra_flags.remove(flag)
if found_flags != set(flags):
raise SoftwarePackageError("Score-P installation at '%s' was not configured with flags %s" %
(self.install_prefix, ' '.join(flags)))
if extra_flags:
raise SoftwarePackageError("Score-P installation at '%s' was configured with extra flags %s" %
(self.install_prefix, ' '.join(extra_flags)))
def xml_event_info(self):
if not self._xml_event_info:
self.install()
xml_event_info = util.get_command_output(os.path.join(self.bin_path, 'papi_xml_event_info'))
self._xml_event_info = ElementTree.fromstring(xml_event_info)
return self._xml_event_info
def probe(cls, absolute_path, candidates=None):
"""Determine the compiler family of a given command.
Executes the command with :any:`version_flags` from all families
and compares the output against :any:`family_regex`.
Args:
absolute_path (str): Absolute path to a compiler command.
candidates (list): If present, a list of families that are most likely.
Raises:
ConfigurationError: Compiler family could not be determined.
Returns:
_CompilerFamily: The compiler's family.
"""
try:
return cls._probe_cache[absolute_path]
except KeyError:
pass
LOGGER.debug("Probing compiler '%s' to discover compiler family", absolute_path)
messages = []
stdout = None
# Settle down pylint... the __instances__ member is created by __new__
# pylint: disable=no-member
if candidates:
candidate_kbases = {candidate.kbase for candidate in candidates}
families = candidates + [inst for inst in cls.__instances__
if inst not in candidates and inst.kbase in candidate_kbases]
else:
families = cls.__instances__
basename = os.path.basename(absolute_path)
with_regex, without_regex = [], []
for family in families:
if basename in family.commands:
(with_regex if family.family_regex else without_regex).append(family)
for family in with_regex:
cmd = [absolute_path] + family.version_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError as err:
messages.append(err.output)
LOGGER.debug("%s returned %d: %s", cmd, err.returncode, err.output)
# Keep going: Cray compilers return nonzero on version flag
if stdout:
if re.search(family.family_regex, stdout, re.MULTILINE):
if family.show_wrapper_flags:
cmd = [absolute_path] + family.show_wrapper_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError as err:
messages.append(err.output)
LOGGER.debug("%s returned %d: %s", cmd, err.returncode, err.output)
if stdout:
if re.search(family.family_regex, stdout, re.MULTILINE):
LOGGER.debug("'%s' is a %s compiler", absolute_path, family.name)
cls._probe_cache[absolute_path] = family
return family
else:
LOGGER.debug("'%s' is not a %s compiler", absolute_path, family.name)
else:
LOGGER.debug("'%s' is a %s compiler", absolute_path, family.name)
cls._probe_cache[absolute_path] = family
return family
else:
LOGGER.debug("'%s' is not a %s compiler", absolute_path, family.name)
if len(without_regex) > 0:
guess = without_regex[0]
if len(without_regex) > 1:
LOGGER.warning(("Assuming '%s' is a %s compiler but it could be to any of these: %s\n"
"If this assumption is incorrect then you should manually specify your compilers"),
absolute_path, guess.name, ', '.join([family.name for family in without_regex]))
return guess
raise ConfigurationError("Cannot determine compiler family: %s" % '\n'.join(messages))