def configure(self, flags):
from taucmdr.cf.platforms import DARWIN, IBM_BGP, IBM_BGQ, INTEL_KNC
flags.extend(['--disable-nls', '--disable-werror'])
for var in 'CPP', 'CC', 'CXX', 'FC', 'F77', 'F90':
os.environ.pop(var, None)
if self.target_os is DARWIN:
flags.append('CFLAGS=-Wno-error=unused-value -Wno-error=deprecated-declarations -fPIC')
flags.append('CXXFLAGS=-Wno-error=unused-value -Wno-error=deprecated-declarations -fPIC')
else:
flags.append('CFLAGS=-fPIC')
flags.append('CXXFLAGS=-fPIC')
if self.target_arch is IBM_BGP:
flags.append('CC=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc-bgp-linux-gcc')
flags.append('CXX=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc-bgp-linux-g++')
elif self.target_arch is IBM_BGQ:
flags.append('CC=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc64-bgq-linux-gcc')
flags.append('CXX=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc64-bgq-linux-g++')
elif self.target_arch.is_ibm():
flags.append('--disable-largefile')
elif self.target_arch is INTEL_KNC:
k1om_ar = util.which('x86_64-k1om-linux-ar')
if not k1om_ar:
for path in glob.glob('/usr/linux-k1om-*'):
k1om_ar = util.which(os.path.join(path, 'bin', 'x86_64-k1om-linux-ar'))
if k1om_ar:
break
else:
raise ConfigurationError("Cannot find KNC native compilers in /usr/linux-k1om-*")
os.environ['PATH'] = os.pathsep.join((os.path.dirname(k1om_ar), os.environ['PATH']))
flags.append('--host=x86_64-k1om-linux')
return super(BinutilsInstallation, self).configure(flags)
def __init__(self, family):
self.family = family
self.members = {}
LOGGER.debug("Detecting %s compiler installation", family.name)
for role, info_list in family.members.iteritems():
for info in info_list:
absolute_path = util.which(info.command)
if absolute_path:
LOGGER.debug("%s %s compiler is '%s'", family.name,
info.role.language, absolute_path)
try:
installed = InstalledCompiler(absolute_path, info)
except ConfigurationError as err:
LOGGER.warning(err)
continue
self.members.setdefault(role, []).append(installed)
if not self.members:
cmds = [
info.command for info_list in family.members.itervalues()
for info in info_list
]
raise ConfigurationError(
"%s %s not found." %
(self.family.name, self.family.kbase.description),
"Check that these commands are in your PATH: %s" %
', '.join(cmds))
def check_compiler(self, compiler_cmd, compiler_args):
"""Checks a compiler command its arguments for compatibility with this target configuration.
Checks that the given compiler matches at least one, **but possibly more**, of the compilers
used in the target. Also performs any special checkes for invalid compiler arguments,
e.g. -mmic is only for native KNC.
If the given compiler command and arguments are compatible with this target then information about
matching compiler installations is returned as a list of n :any:`InstalledCompiler` instances.
Args:
compiler_cmd (str): The compiler command as passed by the user.
compiler_args (list): Compiler command line arguments.
Returns:
list: Information about matching installed compilers as :any:`Compiler` instances.
Raises:
ConfigurationError: The compiler or command line arguments are incompatible with this target.
"""
if '-mmic' in compiler_args and self['host_arch'] != str(INTEL_KNC):
raise ConfigurationError("Host architecture of target '%s' is '%s'"
" but the '-mmic' compiler argument requires '%s'" %
(self['name'], self['host_arch'], INTEL_KNC),
"Select a different target",
"Create a new target with host architecture '%s'" % INTEL_KNC)
compiler_ctrl = Compiler.controller(self.storage)
absolute_path = util.which(compiler_cmd)
compiler_cmd = os.path.basename(compiler_cmd)
found = []
known_compilers = [comp for comp in self.compilers().itervalues()]
for info in Knowledgebase.find_compiler(command=compiler_cmd):
try:
compiler_record = self.populate(info.role.keyword)
except KeyError:
# Target was not configured with a compiler in this role
continue
compiler_path = compiler_record['path']
if (absolute_path and (compiler_path == absolute_path) or
(not absolute_path and (os.path.basename(compiler_path) == compiler_cmd))):
found.append(compiler_record)
else:
# Target was configured with a wrapper compiler so check if that wrapper wraps this compiler
while 'wrapped' in compiler_record:
compiler_record = compiler_ctrl.one(compiler_record['wrapped'])
known_compilers.append(compiler_record.installation())
compiler_path = compiler_record['path']
if (absolute_path and (compiler_path == absolute_path) or
(not absolute_path and (os.path.basename(compiler_path) == compiler_cmd))):
found.append(compiler_record)
break
if not found:
parts = ["No compiler in target '%s' matches '%s'." % (self['name'], absolute_path or compiler_cmd),
"The known compiler commands are:"]
parts.extend(' %s (%s)' % (comp.absolute_path, comp.info.short_descr) for comp in known_compilers)
hints = ("Try one of the valid compiler commands",
"Create and select a new target configuration that uses the '%s' compiler" % (absolute_path or compiler_cmd),
"Check loaded modules and the PATH environment variable")
raise ConfigurationError('\n'.join(parts), *hints)
return found
def _perform_bluegene(self, expr, trial, cmd, cwd, env):
if os.path.basename(cmd[0]) != 'qsub':
raise TrialError("At the moment, TAU Commander requires qsub to launch on BlueGene")
# Move TAU environment parameters to the command line
env_parts = {}
for key, val in env.iteritems():
if key not in os.environ:
env_parts[key] = val.replace(":", r"\:").replace("=", r"\=")
env_str = ':'.join(['%s=%s' % item for item in env_parts.iteritems()])
cmd = [util.which('tau_exec') if c == 'tau_exec' else c for c in cmd]
cmd = [cmd[0], '--env', '"%s"' % env_str] + cmd[1:]
env = dict(os.environ)
try:
self.update({'phase': 'executing'}, trial.eid)
retval = trial.queue_command(expr, cmd, cwd, env)
except:
self.delete(trial.eid)
raise
if retval != 0:
raise TrialError("Failed to add job to the queue.",
"Verify that the right input parameters were specified.",
"Check the program output for error messages.",
"Does the selected application configuration correctly describe this program?",
"Does the selected measurement configuration specifiy the right measurement methods?",
"Does the selected target configuration match the runtime environment?")
else:
LOGGER.info("The job has been added to the queue.")
return retval
def knc_require_k1om(*_):
"""Compatibility checking callback for use with data models.
Requires that the Intel k1om tools be installed if the host architecture is KNC.
Raises:
ConfigurationError: Invalid compiler family specified in target configuration.
"""
k1om_ar = util.which('x86_64-k1om-linux-ar')
if not k1om_ar:
for path in glob.glob('/usr/linux-k1om-*'):
k1om_ar = util.which(os.path.join(path, 'bin', 'x86_64-k1om-linux-ar'))
if k1om_ar:
break
else:
raise ConfigurationError('k1om tools not found', 'Try installing on compute node', 'Install MIC SDK')
def _perform_bluegene(self, expr, trial, cmd, cwd, env):
if os.path.basename(cmd[0]) != 'qsub':
raise TrialError(
"At the moment, TAU Commander requires qsub to launch on BlueGene"
)
# Move TAU environment parameters to the command line
env_parts = {}
for key, val in env.iteritems():
if key not in os.environ:
env_parts[key] = val.replace(":", r"\:").replace("=", r"\=")
env_str = ':'.join(['%s=%s' % item for item in env_parts.iteritems()])
cmd = [util.which('tau_exec') if c == 'tau_exec' else c for c in cmd]
cmd = [cmd[0], '--env', '"%s"' % env_str] + cmd[1:]
env = dict(os.environ)
try:
self.update({'phase': 'executing'}, trial.eid)
retval = trial.queue_command(expr, cmd, cwd, env)
except:
self.delete(trial.eid)
raise
if retval != 0:
raise TrialError(
"Failed to add job to the queue.",
"Verify that the right input parameters were specified.",
"Check the program output for error messages.",
"Does the selected application configuration correctly describe this program?",
"Does the selected measurement configuration specifiy the right measurement methods?",
"Does the selected target configuration match the runtime environment?"
)
else:
LOGGER.info("The job has been added to the queue.")
return retval
def configure(self, flags):
from taucmdr.cf.platforms import DARWIN, IBM_BGP, IBM_BGQ, INTEL_KNC
flags.extend(['--disable-nls', '--disable-werror'])
for var in 'CPP', 'CC', 'CXX', 'FC', 'F77', 'F90':
os.environ.pop(var, None)
if self.target_os is DARWIN:
flags.append(
'CFLAGS=-Wno-error=unused-value -Wno-error=deprecated-declarations -fPIC'
)
flags.append(
'CXXFLAGS=-Wno-error=unused-value -Wno-error=deprecated-declarations -fPIC'
)
else:
flags.append('CFLAGS=-fPIC')
flags.append('CXXFLAGS=-fPIC')
if self.target_arch is IBM_BGP:
flags.append(
'CC=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc-bgp-linux-gcc'
)
flags.append(
'CXX=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc-bgp-linux-g++'
)
elif self.target_arch is IBM_BGQ:
flags.append(
'CC=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc64-bgq-linux-gcc'
)
flags.append(
'CXX=/bgsys/drivers/ppcfloor/gnu-linux/bin/powerpc64-bgq-linux-g++'
)
elif self.target_arch.is_ibm():
flags.append('--disable-largefile')
elif self.target_arch is INTEL_KNC:
k1om_ar = util.which('x86_64-k1om-linux-ar')
if not k1om_ar:
for path in glob.glob('/usr/linux-k1om-*'):
k1om_ar = util.which(
os.path.join(path, 'bin', 'x86_64-k1om-linux-ar'))
if k1om_ar:
break
else:
raise ConfigurationError(
"Cannot find KNC native compilers in /usr/linux-k1om-*"
)
os.environ['PATH'] = os.pathsep.join(
(os.path.dirname(k1om_ar), os.environ['PATH']))
flags.append('--host=x86_64-k1om-linux')
return super(BinutilsInstallation, self).configure(flags)
def is_cray_login(self):
if self is CRAY_CNL:
if util.which('aprun'):
return False
else:
return True
else:
return False
def _probe_wrapper(self):
if not self.info.family.show_wrapper_flags:
return None
LOGGER.debug("Probing %s wrapper '%s'", self.info.short_descr,
self.absolute_path)
cmd = [self.absolute_path] + self.info.family.show_wrapper_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError:
# If this command didn't accept show_wrapper_flags then it's not a compiler wrapper to begin with,
# i.e. another command just happens to be the same as a known compiler command.
raise ConfigurationError(
"'%s' isn't actually a %s since it doesn't accept arguments %s."
% (self.absolute_path, self.info.short_descr,
self.info.family.show_wrapper_flags))
# Assume the longest line starting with a known compiler command is the wrapped compiler followed by arguments.
known_commands = set(info.command for info in _CompilerInfo.all())
for line in sorted(stdout.split('\n'), key=len, reverse=True):
if not line:
continue
parts = line.split()
wrapped_command = parts[0]
wrapped_args = parts[1:]
if os.path.basename(wrapped_command) not in known_commands:
continue
wrapped_absolute_path = util.which(wrapped_command)
if not wrapped_absolute_path:
continue
if wrapped_absolute_path == self.absolute_path:
# A wrapper that wraps itself isn't a wrapper, e.g. compilers that ignore invalid arguments
# when version flags are present.
return None
try:
wrapped = InstalledCompiler.probe(wrapped_command)
except ConfigurationError:
# wrapped_command might not be a real compiler command so keep trying
continue
# The wrapper must be able to perform the same role as the wrapped compiler
role = self.info.role.keyword.split('_')[1:]
wrapped_role = wrapped.info.role.keyword.split('_')[1:]
if role != wrapped_role:
raise ConfigurationError(
"Cannot use '%s' as a %s: wrapped compiler '%s' is a %s" %
(self.command, self.info.short_descr, wrapped.command,
wrapped.info.short_descr))
LOGGER.info("%s '%s' wraps '%s'", self.info.short_descr,
self.absolute_path, wrapped.absolute_path)
try:
self._parse_wrapped_args(wrapped_args)
except IndexError:
LOGGER.warning(
"Unexpected output from compiler wrapper '%s'."
" TAU will attempt to continue but may fail later on.",
self.absolute_path)
return wrapped
return None
def cuda_toolkit_default():
for path in sorted(glob.glob('/usr/local/cuda*')):
if os.path.exists(os.path.join(path, 'bin', 'nvcc')):
return path
nvcc = util.which('nvcc')
if nvcc:
cuda_dir = os.path.dirname(os.path.dirname(nvcc))
if os.path.exists(os.path.join(cuda_dir, 'include', 'cuda.h')):
return cuda_dir
return None
def is_compatible(cmd):
"""Check if this subcommand can work with the given command.
Args:
cmd (str): A command from the command line, e.g. sys.argv[1].
Returns:
bool: True if this subcommand is compatible with `cmd`.
"""
return bool(util.which(cmd))
def is_compatible(cmd):
"""Check if this subcommand can work with the given command.
Args:
cmd (str): A command from the command line, e.g. sys.argv[1].
Returns:
bool: True if this subcommand is compatible with `cmd`.
"""
return bool(util.which(cmd))
def cuda_toolkit_default():
for path in sorted(glob.glob('/usr/local/cuda*')):
if os.path.exists(os.path.join(path, 'bin', 'nvcc')):
return path
nvcc = util.which('nvcc')
if nvcc:
cuda_dir = os.path.dirname(os.path.dirname(nvcc))
if os.path.exists(os.path.join(cuda_dir, 'include', 'cuda.h')):
return cuda_dir
return None
def knc_require_k1om(*_):
"""Compatibility checking callback for use with data models.
Requires that the Intel k1om tools be installed if the host architecture is KNC.
Raises:
ConfigurationError: Invalid compiler family specified in target configuration.
"""
k1om_ar = util.which('x86_64-k1om-linux-ar')
if not k1om_ar:
for path in glob.glob('/usr/linux-k1om-*'):
k1om_ar = util.which(
os.path.join(path, 'bin', 'x86_64-k1om-linux-ar'))
if k1om_ar:
break
else:
raise ConfigurationError('k1om tools not found',
'Try installing on compute node',
'Install MIC SDK')
def parse_launcher_cmd(cls, cmd):
"""Parses a command line to split the launcher command and application commands.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, possibly empty list of application commands).
"""
cmd0 = cmd[0]
launcher_cmd, cmd = cls._separate_launcher_cmd(cmd)
num_exes = len([x for x in cmd if util.which(x)])
assert launcher_cmd or cmd
LOGGER.debug('Launcher: %s', launcher_cmd)
LOGGER.debug('Remainder: %s', cmd)
uses_python = Project.selected().experiment().populate(
)['application'].get_or_default('python')
if uses_python:
if 'python' in cmd[0]:
cmd.remove(cmd[0])
if not launcher_cmd:
if num_exes > 1:
LOGGER.warning(
"Multiple executables were found on the command line but none of them "
"were recognized application launchers. TAU will assume that the application "
"executable is '%s' and subsequent executables are arguments to that command. "
"If this is incorrect, use '--' to separate '%s' and its arguments "
"from the application command, e.g. `mpirun -np 4 -- ./a.out -l hello`",
cmd0, cmd0)
return [], [cmd]
if not cmd:
return launcher_cmd, []
if num_exes <= 1:
return launcher_cmd, [cmd]
elif num_exes > 1:
colons = [i for i, x in enumerate(cmd) if x == ':']
if not colons:
# Recognized launcher with multiple executables but not using ':' syntax.
LOGGER.warning(
"Multiple executables were found on the command line. TAU will assume that "
"the application executable is '%s' and subsequent executables are arguments "
"to that command. If this is incorrect, use ':' to separate each application "
"executable and its arguments, e.g. `mpirun -np 4 ./foo -l : -np 2 ./bar arg1`. "
"Or, use '--' to separate '%s', its arguments, and subsequent executables "
"from the application command, e.g. "
"`mpirun -np 4 numactl -m 1 -- ./a.out -l hello", cmd0,
cmd0)
return launcher_cmd, [cmd]
# Split MPMD command on ':'. Retain ':' as first element of each application command
colons.append(len(cmd))
application_cmds = [cmd[:colons[0]]]
for i, idx in enumerate(colons[:-1]):
application_cmds.append(cmd[idx:colons[i + 1]])
return launcher_cmd, application_cmds
def __init__(self, family):
self.family = family
self.members = {}
LOGGER.debug("Detecting %s compiler installation", family.name)
for role, info_list in family.members.iteritems():
for info in info_list:
absolute_path = util.which(info.command)
if absolute_path:
LOGGER.debug("%s %s compiler is '%s'", family.name, info.role.language, absolute_path)
installed = InstalledCompiler(absolute_path, info)
self.members.setdefault(role, []).append(installed)
if not self.members:
raise ConfigurationError("%s compilers not found." % self.family.name)
def is_compatible(cmd):
"""Check if this subcommand can work with the given command.
Args:
cmd (str): A command from the command line, e.g. sys.argv[1].
Returns:
bool: True if this subcommand is compatible with `cmd`.
"""
uses_python = Project.selected().experiment().populate()['application'].get_or_default('python')
if uses_python and 'python' in cmd:
return True
return bool(util.which(cmd))
def _separate_launcher_cmd(cls, cmd):
"""Separate the launcher command and it's arguments from the application command(s) and arguments.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, Remainder of command line)
Raises:
ConfigurationError: No application config files or executables found after a recognized launcher command.
"""
# If '--' appears in the command then everything before it is a launcher + args
# and everything after is the application + args
try:
idx = cmd.index('--')
except ValueError:
pass
else:
return cmd[:idx], cmd[idx + 1:]
cmd0 = cmd[0]
for launcher, appfile_flags in PROGRAM_LAUNCHERS.iteritems():
if launcher not in cmd0:
continue
# No '--' to indicate start of application, so look for first executable
for idx, exe in enumerate(cmd[1:], 1):
if util.which(exe):
return cmd[:idx], cmd[idx:]
# No exectuables, so look for application config file
if appfile_flags:
for i, arg in enumerate(cmd[1:], 1):
try:
arg, appfile = arg.split('=')
except ValueError:
try:
appfile = cmd[i + 1]
except IndexError:
# Reached the end of the command line without finding an application config file
break
if arg in appfile_flags and util.path_accessible(appfile):
return cmd, []
raise ConfigurationError(
("TAU is having trouble parsing the command line: no executable "
"commands or %s application files were found after "
"the launcher command '%s'") % (cmd0, cmd0),
"Check that the command is correct. Does it work without TAU?",
("Use '--' to seperate '%s' and its arguments from the application "
"command, e.g. `mpirun -np 4 -- ./a.out -l hello`" % cmd0))
# No launcher command, just an application command
return [], cmd
def _separate_launcher_cmd(cls, cmd):
"""Separate the launcher command and it's arguments from the application command(s) and arguments.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, Remainder of command line)
Raises:
ConfigurationError: No application config files or executables found after a recognized launcher command.
"""
# If '--' appears in the command then everything before it is a launcher + args
# and everything after is the application + args
try:
idx = cmd.index('--')
except ValueError:
pass
else:
return cmd[:idx], cmd[idx+1:]
cmd0 = cmd[0]
for launcher, appfile_flags in PROGRAM_LAUNCHERS.iteritems():
if launcher not in cmd0:
continue
# No '--' to indicate start of application, so look for first executable
for idx, exe in enumerate(cmd[1:], 1):
if util.which(exe):
return cmd[:idx], cmd[idx:]
# No exectuables, so look for application config file
if appfile_flags:
for i, arg in enumerate(cmd[1:], 1):
try:
arg, appfile = arg.split('=')
except ValueError:
try:
appfile = cmd[i+1]
except IndexError:
# Reached the end of the command line without finding an application config file
break
if arg in appfile_flags and util.path_accessible(appfile):
return cmd, []
raise ConfigurationError(("TAU is having trouble parsing the command line: no executable "
"commands or %s application files were found after "
"the launcher command '%s'") % (cmd0, cmd0),
"Check that the command is correct. Does it work without TAU?",
("Use '--' to seperate '%s' and its arguments from the application "
"command, e.g. `mpirun -np 4 -- ./a.out -l hello`" % cmd0))
# No launcher command, just an application command
return [], cmd
def _probe_wrapper(self):
if not self.info.family.show_wrapper_flags:
return None
LOGGER.debug("Probing %s wrapper '%s'", self.info.short_descr, self.absolute_path)
cmd = [self.absolute_path] + self.info.family.show_wrapper_flags
try:
stdout = util.get_command_output(cmd)
except CalledProcessError:
# If this command didn't accept show_wrapper_flags then it's not a compiler wrapper to begin with,
# i.e. another command just happens to be the same as a known compiler command.
raise ConfigurationError("'%s' isn't actually a %s since it doesn't accept arguments %s." %
(self.absolute_path, self.info.short_descr, self.info.family.show_wrapper_flags))
# Assume the longest line starting with a known compiler command is the wrapped compiler followed by arguments.
known_commands = set(info.command for info in _CompilerInfo.all())
for line in sorted(stdout.split('\n'), key=len, reverse=True):
if not line:
continue
parts = line.split()
wrapped_command = parts[0]
wrapped_args = parts[1:]
if os.path.basename(wrapped_command) not in known_commands:
continue
wrapped_absolute_path = util.which(wrapped_command)
if not wrapped_absolute_path:
continue
if wrapped_absolute_path == self.absolute_path:
# A wrapper that wraps itself isn't a wrapper, e.g. compilers that ignore invalid arguments
# when version flags are present.
return None
try:
wrapped = InstalledCompiler.probe(wrapped_command)
except ConfigurationError:
# wrapped_command might not be a real compiler command so keep trying
continue
# The wrapper must be able to perform the same role as the wrapped compiler
role = self.info.role.keyword.split('_')[1:]
wrapped_role = wrapped.info.role.keyword.split('_')[1:]
if role != wrapped_role:
raise ConfigurationError("Cannot use '%s' as a %s: wrapped compiler '%s' is a %s" %
(self.command, self.info.short_descr,
wrapped.command, wrapped.info.short_descr))
LOGGER.info("%s '%s' wraps '%s'", self.info.short_descr, self.absolute_path, wrapped.absolute_path)
try:
self._parse_wrapped_args(wrapped_args)
except IndexError:
LOGGER.warning("Unexpected output from compiler wrapper '%s'."
" TAU will attempt to continue but may fail later on.", self.absolute_path)
return wrapped
return None
def _get_cmake(self):
cmake = util.which('cmake')
if not cmake:
raise ConfigurationError("'cmake' not found in PATH.")
try:
stdout = util.get_command_output([cmake, '--version'])
except (CalledProcessError, OSError) as err:
raise ConfigurationError("Failed to get CMake version: %s" % err)
for line in stdout.split('\n'):
if 'cmake version' in line:
verstr = (line.split('cmake version ')[1]).split('-')[0]
version = tuple(int(x) for x in verstr.split('.'))
if version < (2, 8):
raise ConfigurationError("CMake version 2.8 or higher required.")
break
else:
LOGGER.warning("Cannot determine CMake version. CMake 2.8 or higher is required.")
return cmake
def parse_launcher_cmd(cls, cmd):
"""Parses a command line to split the launcher command and application commands.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, possibly empty list of application commands).
"""
cmd0 = cmd[0]
launcher_cmd, cmd = cls._separate_launcher_cmd(cmd)
num_exes = len([x for x in cmd if util.which(x)])
assert launcher_cmd or cmd
LOGGER.debug('Launcher: %s', launcher_cmd)
LOGGER.debug('Remainder: %s', cmd)
if not launcher_cmd:
if num_exes > 1:
LOGGER.warning("Multiple executables were found on the command line but none of them "
"were recognized application launchers. TAU will assume that the application "
"executable is '%s' and subsequent executables are arguments to that command. "
"If this is incorrect, use '--' to separate '%s' and its arguments "
"from the application command, e.g. `mpirun -np 4 -- ./a.out -l hello`", cmd0, cmd0)
return [], [cmd]
if not cmd:
return launcher_cmd, []
if num_exes <= 1:
return launcher_cmd, [cmd]
elif num_exes > 1:
colons = [i for i, x in enumerate(cmd) if x == ':']
if not colons:
# Recognized launcher with multiple executables but not using ':' syntax.
LOGGER.warning("Multiple executables were found on the command line. TAU will assume that "
"the application executable is '%s' and subsequent executables are arguments "
"to that command. If this is incorrect, use ':' to separate each application "
"executable and its arguments, e.g. `mpirun -np 4 ./foo -l : -np 2 ./bar arg1`", cmd0)
return launcher_cmd, [cmd]
# Split MPMD command on ':'. Retain ':' as first element of each application command
colons.append(len(cmd))
application_cmds = [cmd[:colons[0]]]
for i, idx in enumerate(colons[:-1]):
application_cmds.append(cmd[idx:colons[i+1]])
return launcher_cmd, application_cmds
def probe(cls, command, family=None, role=None):
"""Probe the system to discover information about an installed compiler.
Args:
command (str): Absolute or relative path to an installed compiler command.
family (_CompilerFamily): Installed compiler's family if known, None otherwise.
role (_CompilerRole): Installed compiler's role if known, None otherwise.
Raises:
ConfigurationError: Unknown compiler command or not enough information given to perform the probe.
Returns:
InstalledCompiler: A new InstalledCompiler instance describing the compiler.
"""
assert isinstance(command, basestring)
assert isinstance(family, _CompilerFamily) or family is None
assert isinstance(role, _CompilerRole) or role is None
absolute_path = util.which(command)
if not absolute_path:
raise ConfigurationError("Compiler '%s' not found on PATH" %
command)
command = os.path.basename(absolute_path)
LOGGER.debug(
"Probe: command='%s', abspath='%s', family='%s', role='%s'",
command, absolute_path, family, role)
# Try to identify without probing
info_list = _CompilerInfo.find(command, family, role)
if len(info_list) == 1:
return InstalledCompiler(absolute_path, info_list[0])
# Probe and try again
family = _CompilerFamily.probe(absolute_path,
[info.family for info in info_list])
info_list = _CompilerInfo.find(command, family, role)
if len(info_list) == 1:
return InstalledCompiler(absolute_path, info_list[0])
msg_parts = [
"Unable to identify ", (family.name + " " if family else ""),
(role.language + " " if role else ""),
"compiler '%s'" % absolute_path
]
raise ConfigurationError(''.join(msg_parts))
def _get_cmake(self):
cmake = util.which('cmake')
if not cmake:
raise ConfigurationError("'cmake' not found in PATH.")
try:
stdout = util.get_command_output([cmake, '--version'])
except (CalledProcessError, OSError) as err:
raise ConfigurationError("Failed to get CMake version: %s" % err)
for line in stdout.split('\n'):
if 'cmake version' in line:
verstr = (line.split('cmake version ')[1]).split('-')[0]
version = tuple(int(x) for x in verstr.split('.'))
if version < (2, 8):
raise ConfigurationError(
"CMake version 2.8 or higher required.")
break
else:
LOGGER.warning(
"Cannot determine CMake version. CMake 2.8 or higher is required."
)
return cmake
def probe(cls, command, family=None, role=None):
"""Probe the system to discover information about an installed compiler.
Args:
command (str): Absolute or relative path to an installed compiler command.
family (_CompilerFamily): Installed compiler's family if known, None otherwise.
role (_CompilerRole): Installed compiler's role if known, None otherwise.
Raises:
ConfigurationError: Unknown compiler command or not enough information given to perform the probe.
Returns:
InstalledCompiler: A new InstalledCompiler instance describing the compiler.
"""
assert isinstance(command, basestring)
assert isinstance(family, _CompilerFamily) or family is None
assert isinstance(role, _CompilerRole) or role is None
absolute_path = util.which(command)
if not absolute_path:
raise ConfigurationError("Compiler '%s' not found on PATH" % command)
command = os.path.basename(absolute_path)
LOGGER.debug("Probe: command='%s', abspath='%s', family='%s', role='%s'",
command, absolute_path, family, role)
# Try to identify without probing
info_list = _CompilerInfo.find(command, family, role)
if len(info_list) == 1:
return InstalledCompiler(absolute_path, info_list[0])
# Probe and try again
family = _CompilerFamily.probe(absolute_path, [info.family for info in info_list])
info_list = _CompilerInfo.find(command, family, role)
if len(info_list) == 1:
return InstalledCompiler(absolute_path, info_list[0])
msg_parts = ["Unable to identify ",
(family.name + " " if family else ""),
(role.language + " " if role else ""),
"compiler '%s'" % absolute_path]
raise ConfigurationError(''.join(msg_parts))
class CreateLauncherTest(tests.TestCase):
"""Tests for :any:`trial.create` with an application launcher.
https://github.com/ParaToolsInc/taucmdr/issues/210
"""
def test_foo_launcher_simple(self):
self.reset_project_storage()
self.copy_testfile('foo_launcher')
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, ['./foo_launcher', './a.out'])
self.assertFalse(stderr)
self.assertIn("Multiple executables were found", stdout)
self.assertIn("executable is './foo_launcher'", stdout)
self.assertIn("FOO LAUNCHER\nDone", stdout)
self.assertRegexpMatches(stdout, r'tau_exec .* ./foo_launcher ./a.out')
def test_launcher_flag(self):
self.reset_project_storage()
self.copy_testfile('foo_launcher')
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, ['./foo_launcher', '--', './a.out'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertIn("FOO LAUNCHER\nDone", stdout)
self.assertRegexpMatches(stdout, r'./foo_launcher tau_exec .* ./a.out')
def test_invalid_exe(self):
self.reset_project_storage()
self.copy_testfile('foo_launcher')
stdout, stderr = self.assertNotCommandReturnValue(
0, trial_create_cmd, ['./foo_launcher', './invalid'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertIn("FOO LAUNCHER", stdout)
self.assertRegexpMatches(stdout,
r'tau_exec .* ./foo_launcher ./invalid')
@tests.skipUnless(util.which('mpirun'), "mpirun required for this test")
@tests.skipUnlessHaveCompiler(MPI_CC)
def test_mpirun(self):
self.reset_project_storage(['--mpi'])
self.assertManagedBuild(0, MPI_CC, ['-DTAU_MPI'], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, ['mpirun', '-np', '4', './a.out'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertRegexpMatches(stdout, r'mpirun -np 4 tau_exec .* ./a.out')
@tests.skipUnless(util.which('mpirun'), "mpirun required for this test")
@tests.skipUnlessHaveCompiler(MPI_CC)
def test_mpirun_with_flag(self):
self.reset_project_storage(['--mpi'])
self.assertManagedBuild(0, MPI_CC, ['-DTAU_MPI'], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, ['mpirun', '-np', '4', '--', './a.out'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertIn("produced 4 profile files", stdout)
self.assertRegexpMatches(stdout, r'mpirun -np 4 tau_exec .* ./a.out')
@tests.skipUnless(util.which('mpirun'), "mpirun required for this test")
@tests.skipUnlessHaveCompiler(MPI_CC)
def test_mpirun_mpmd(self):
self.reset_project_storage(['--mpi'])
self.assertManagedBuild(0, MPI_CC, ['-DTAU_MPI'], 'matmult.c')
shutil.copy('./a.out', './b.out')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd,
['mpirun', '-np', '2', './a.out', ':', '-np', '2', './b.out'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertIn("produced 4 profile files", stdout)
self.assertRegexpMatches(
stdout,
r'mpirun -np 2 tau_exec .* ./a.out : -np 2 tau_exec .* ./b.out')
@tests.skipUnless(util.which('cafrun'), "cafrun required for this test")
@tests.skipUnlessHaveCompiler(CAF_FC)
def test_cafrun(self):
self.reset_project_storage([
'--caf', '--caf-fc', 'caf', '--source-inst', 'never',
'--compiler-inst', 'always', '--mpi'
])
self.assertManagedBuild(0, CAF_FC, [], 'blockmatrix-coarray.f90')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, ['cafrun', '-np', '9', './a.out'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertRegexpMatches(stdout, r'cafrun -np 9 tau_exec .* ./a.out')
@tests.skipUnless(util.which('cafrun'), "cafrun required for this test")
@tests.skipUnlessHaveCompiler(CAF_FC)
def test_cafrun_with_flag(self):
self.reset_project_storage([
'--caf', '--caf-fc', 'caf', '--source-inst', 'never',
'--compiler-inst', 'always', '--mpi'
])
self.assertManagedBuild(0, CAF_FC, [], 'blockmatrix-coarray.f90')
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, ['cafrun', '-np', '9', '--', './a.out'])
self.assertFalse(stderr)
self.assertNotIn("Multiple executables were found", stdout)
self.assertNotIn("executable is './foo_launcher'", stdout)
self.assertIn("produced 9 profile files", stdout)
self.assertRegexpMatches(stdout, r'cafrun -np 9 tau_exec .* ./a.out')
class CreateTest(tests.TestCase):
"""Tests for :any:`trial.create`."""
@tests.skipIf(HOST_ARCH.is_bluegene(), "Test skipped on BlueGene")
def test_create(self):
self.reset_project_storage()
self.assertManagedBuild(0, CC, [], 'hello.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd,
['./a.out'])
self.assertIn('BEGIN targ1-app1', stdout)
self.assertIn('END targ1-app1', stdout)
self.assertIn('Trial 0 produced', stdout)
self.assertIn('profile files', stdout)
self.assertFalse(stderr)
def test_h_arg(self):
self.reset_project_storage()
stdout, _ = self.assertCommandReturnValue(0, trial_create_cmd, ['-h'])
self.assertIn('Show this help message and exit', stdout)
def test_help_arg(self):
self.reset_project_storage()
stdout, _ = self.assertCommandReturnValue(0, trial_create_cmd,
['--help'])
self.assertIn('Show this help message and exit', stdout)
def test_no_time_metric(self):
self.reset_project_storage()
argv = [
'meas_no_time', '--metrics', 'PAPI_FP_INS', '--source-inst',
'never'
]
self.assertCommandReturnValue(0, measurement_create_cmd, argv)
argv = [
'exp2', '--target', 'targ1', '--application', 'app1',
'--measurement', 'meas_no_time'
]
self.assertCommandReturnValue(0, experiment_create_cmd, argv)
self.assertCommandReturnValue(0, experiment_select_cmd, ['exp2'])
self.assertManagedBuild(0, CC, [], 'hello.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd,
['./a.out'])
self.assertIn("TAU_METRICS=TIME,", stdout)
self.assertFalse(stderr)
@tests.skipIf(HOST_OS is DARWIN, "Sampling not supported on Darwin")
def test_heap_usage_memory_alloc_sample(self):
"""https://github.com/ParaToolsInc/taucmdr/issues/14"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, measurement_edit_cmd,
['sample', '--heap-usage', '--memory-alloc'])
self.assertIn("Updated measurement 'sample'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd,
['sample'])
self.assertIn("Selected experiment 'targ1-app1-sample'", stdout)
self.assertFalse(stderr)
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd,
['./a.out'])
self.assertIn("Trial 0 produced 1 profile files", stdout)
self.assertIn("TAU_SHOW_MEMORY_FUNCTIONS=1", stdout)
self.assertIn("TAU_TRACK_HEAP=1", stdout)
self.assertFalse(stderr)
self.assertInLastTrialData(
"<attribute><name>TAU_SHOW_MEMORY_FUNCTIONS</name><value>on</value></attribute>"
)
self.assertInLastTrialData(
"<attribute><name>TAU_TRACK_HEAP</name><value>on</value></attribute>"
)
self.assertInLastTrialData("Heap Memory Used (KB) at Entry")
self.assertInLastTrialData("Heap Memory Used (KB) at Exit")
self.assertInLastTrialData("compute_interchange")
self.assertInLastTrialData("compute")
# TAU bug: the dynamic malloc wrapper (e.g. tau_exec -memory) doesn't always capture malloc().
#self.assertInLastTrialData("Heap Allocate")
#self.assertInLastTrialData("malloc")
def test_heap_usage_memory_alloc_profile(self):
"""https://github.com/ParaToolsInc/taucmdr/issues/14"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, measurement_edit_cmd,
['profile', '--heap-usage', '--memory-alloc'])
self.assertIn("Updated measurement 'profile'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd,
['profile'])
self.assertIn("Selected experiment 'targ1-app1-profile'", stdout)
self.assertFalse(stderr)
meas = Project.selected().experiment().populate('measurement')
self.assertTrue(meas['heap_usage'])
self.assertTrue(meas['memory_alloc'])
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd,
['./a.out'])
self.assertIn("Trial 0 produced 1 profile files", stdout)
self.assertIn("TAU_SHOW_MEMORY_FUNCTIONS=1", stdout)
self.assertIn("TAU_TRACK_HEAP=1", stdout)
self.assertFalse(stderr)
self.assertInLastTrialData(
"<attribute><name>TAU_SHOW_MEMORY_FUNCTIONS</name><value>on</value></attribute>"
)
self.assertInLastTrialData(
"<attribute><name>TAU_TRACK_HEAP</name><value>on</value></attribute>"
)
self.assertInLastTrialData("Heap Memory Used (KB) at Entry")
self.assertInLastTrialData("Heap Memory Used (KB) at Exit")
self.assertInLastTrialData("Heap Allocate")
self.assertInLastTrialData("compute_interchange")
self.assertInLastTrialData("compute")
self.assertInLastTrialData("malloc")
def sample_resolution_helper(self, option):
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, measurement_edit_cmd, ['sample', '--sample-resolution', option])
self.assertIn("Updated measurement 'sample'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd,
['sample'])
self.assertIn("Selected experiment 'targ1-app1-sample'", stdout)
self.assertFalse(stderr)
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd,
['./a.out'])
self.assertIn("Trial 0 produced 1 profile files", stdout)
self.assertIn("TAU_EBS_RESOLUTION=" + option, stdout)
self.assertFalse(stderr)
def test_sample_resolution_file(self):
self.sample_resolution_helper('file')
def test_sample_resolution_function(self):
self.sample_resolution_helper('function')
def test_sample_resolution_line(self):
self.sample_resolution_helper('line')
@tests.skipIf(HOST_OS is DARWIN, "Sampling not supported on Darwin")
def test_system_load_sample(self):
"""Test TAU_TRACK_LOAD w/ sampling"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, measurement_edit_cmd, ['sample', '--system-load'])
self.assertIn("Updated measurement 'sample'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd,
['sample'])
self.assertIn("Selected experiment 'targ1-app1-sample'", stdout)
self.assertFalse(stderr)
def test_system_load_profile(self):
"""Test TAU_TRACK_LOAD w/ profiling"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, measurement_edit_cmd, ['profile', '--system-load'])
self.assertIn("Updated measurement 'profile'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd,
['profile'])
self.assertIn("Selected experiment 'targ1-app1-profile'", stdout)
self.assertFalse(stderr)
def test_tau_dir(self):
"""Test --tau_dir option"""
self.reset_project_storage()
self.assertManagedBuild(0, CC, [], 'hello.c')
test_dir = os.getcwd()
path = tempfile.mkdtemp()
os.chdir(path)
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd, [test_dir + '/a.out', '--tau-dir', test_dir])
self.assertIn('BEGIN targ1-app1', stdout)
self.assertIn('END targ1-app1', stdout)
self.assertIn('Trial 0 produced', stdout)
self.assertIn('profile files', stdout)
self.assertFalse(stderr)
os.chdir(test_dir)
shutil.rmtree(path)
def test_description(self):
"""Test --description option"""
self.reset_project_storage()
self.assertManagedBuild(0, CC, [], 'hello.c')
self.assertCommandReturnValue(
0, trial_create_cmd, ['--description', 'test desc', './a.out'])
stdout, stderr = self.assertCommandReturnValue(0, trial_list_cmd, [])
self.assertIn('test desc', stdout)
@tests.skipUnless(util.which('python'),
"Python 2 or 3 required for this test")
def test_run_python(self):
self.reset_project_storage(
['--python', 'T', '--python-interpreter', 'python'])
self.copy_testfile('firstprime.py')
test_dir = os.getcwd()
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd,
['python', os.path.join(test_dir, 'firstprime.py')])
self.assertIn('Trial 0 produced', stdout)
self.assertIn('profile files', stdout)
self.assertFalse(stderr)
self.assertInLastTrialData("firstPrimeAfter")
@tests.skipUnless(util.which('python2'), "Python 2 required for this test")
def test_run_python2(self):
self.reset_project_storage(
['--python', 'T', '--python-interpreter', 'python2'])
self.copy_testfile('firstprime.py')
test_dir = os.getcwd()
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd,
['python2', os.path.join(test_dir, 'firstprime.py')])
self.assertIn('Trial 0 produced', stdout)
self.assertIn('profile files', stdout)
self.assertFalse(stderr)
self.assertInLastTrialData("firstPrimeAfter")
@tests.skipUnless(util.which('python3'), "Python 3 required for this test")
def test_run_python3(self):
self.reset_project_storage(
['--python', 'T', '--python-interpreter', 'python3'])
self.copy_testfile('firstprime.py')
test_dir = os.getcwd()
stdout, stderr = self.assertCommandReturnValue(
0, trial_create_cmd,
['python3', os.path.join(test_dir, 'firstprime.py')])
self.assertIn('Trial 0 produced', stdout)
self.assertIn('profile files', stdout)
self.assertFalse(stderr)
self.assertInLastTrialData("firstPrimeAfter")
class CreateTest(tests.TestCase):
@tests.skipUnless(util.which('pgcc'),
"PGI compilers required for this test")
def test_pgi(self):
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, target_create_cmd, ['test_targ', '--compilers', 'PGI'])
self.assertIn("Added target 'test_targ' to project configuration",
stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0,
measurement_create_cmd,
['meas_PGI'])
self.assertIn("Added measurement 'meas_PGI' to project configuration",
stdout)
self.assertFalse(stderr)
argv = [
'exp2', '--target', 'test_targ', '--application', 'app1',
'--measurement', 'meas_PGI'
]
_, stderr = self.assertCommandReturnValue(0, experiment_create_cmd,
argv)
self.assertFalse(stderr)
def test_h_arg(self):
self.reset_project_storage()
stdout, _ = self.assertCommandReturnValue(0, experiment_create_cmd,
['-h'])
self.assertIn('Show this help message and exit', stdout)
def test_help_arg(self):
self.reset_project_storage()
stdout, _ = self.assertCommandReturnValue(0, experiment_create_cmd,
['--help'])
self.assertIn('Show this help message and exit', stdout)
def test_invalid_targ_name(self):
self.reset_project_storage()
argv = [
'exp2', '--target', 'targ_err', '--application', 'app1',
'--measurement', 'sample'
]
stdout, stderr = self.assertNotCommandReturnValue(
0, experiment_create_cmd, argv)
self.assertNotIn('CRITICAL', stdout)
self.assertIn('error', stderr)
def test_invalid_app_name(self):
self.reset_project_storage()
argv = [
'exp2', '--target', 'targ1', '--application', 'app_err',
'--measurement', 'sample'
]
stdout, stderr = self.assertNotCommandReturnValue(
0, experiment_create_cmd, argv)
self.assertNotIn('CRITICAL', stdout)
self.assertIn('error', stderr)
def test_invalid_meas_name(self):
self.reset_project_storage()
argv = [
'exp2', '--target', 'targ1', '--application', 'app1',
'--measurement', 'meas_err'
]
stdout, stderr = self.assertNotCommandReturnValue(
0, experiment_create_cmd, argv)
self.assertNotIn('CRITICAL', stdout)
self.assertIn('error', stderr)
def test_ompt(self):
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, application_create_cmd, ['test_app', '--openmp', 'T'])
self.assertIn("Added application 'test_app' to project configuration",
stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(
0, measurement_create_cmd, ['meas_ompt', '--openmp', 'ompt'])
self.assertIn("Added measurement 'meas_ompt' to project configuration",
stdout)
self.assertFalse(stderr)
argv = [
'exp2', '--target', 'targ1', '--application', 'test_app',
'--measurement', 'meas_ompt'
]
_, stderr = self.assertCommandReturnValue(0, experiment_create_cmd,
argv)
self.assertFalse(stderr)
@tests.skipIf(HOST_OS is DARWIN, "No 'sample' measurement on Darwin.")
def test_new_project(self):
"""https://github.com/ParaToolsInc/taucmdr/issues/29"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(0, project_create_cmd,
['test_proj'])
self.assertIn("Created a new project named 'test_proj'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(
0, project_edit_cmd,
['test_proj', '--add-measurements', 'profile', 'sample'])
self.assertIn("Added measurement 'profile' to project configuration",
stdout)
self.assertIn("Added measurement 'sample' to project configuration",
stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(
0, project_edit_cmd, ['test_proj', '--add-application', 'app1'])
self.assertIn("Added application 'app1' to project configuration",
stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(
0, project_edit_cmd, ['test_proj', '--add-target', 'targ1'])
self.assertIn("Added target 'targ1' to project configuration", stdout)
self.assertFalse(stderr)
_, stderr = self.assertCommandReturnValue(0, project_select_cmd,
['test_proj'])
self.assertFalse(stderr)
argv = [
'exp2', '--target', 'targ1', '--application', 'app1',
'--measurement', 'sample'
]
stdout, stderr = self.assertCommandReturnValue(0,
experiment_create_cmd,
argv)
self.assertIn("Created a new experiment", stdout)
self.assertFalse(stderr)
def _parse_tau_makefile(self, args):
# Parsing a TAU Makefile is a really hairy operation, so let's lift the limits
# pylint: disable=too-many-statements,too-many-locals
makefile = args.forced_makefile
if not util.path_accessible(makefile):
self.parser.error("Invalid TAU makefile: %s" % makefile)
tau_arch_name = os.path.basename(
os.path.dirname(os.path.dirname(makefile)))
matches = [
arch for arch in TauMagic.all() if arch.name == tau_arch_name
]
if len(matches) == 1:
tau_arch = matches[0]
elif not matches:
raise ConfigurationError(
"TAU Makefile '%s' targets an unrecognized TAU architecture: %s"
% (makefile, tau_arch_name))
else:
for arch in matches:
if arch.architecture == HOST_ARCH and arch.operating_system == HOST_OS:
tau_arch = arch
break
else:
parts = [
"TAU Makefile '%s' targets an ambiguous TAU architecture: %s"
% (makefile, tau_arch_name), "It could be any of these:"
]
parts.extend([
" - %s on %s" %
(arch.operating_system.name, arch.architecture.name)
for arch in matches
])
raise ConfigurationError("\n".join(parts))
self.logger.info(
"Parsing TAU Makefile '%s' to populate command line arguments:",
makefile)
args.host_arch = tau_arch.architecture.name
self.logger.info(" --host-arch='%s'", args.host_arch)
args.host_os = tau_arch.operating_system.name
self.logger.info(" --host-os='%s'", args.host_os)
args.tau_source = os.path.abspath(
os.path.join(os.path.dirname(makefile), '..', '..'))
self.logger.info(" --taucmdr='%s'", args.tau_source)
with open(makefile, 'r') as fin:
compiler_parts = ("FULL_CC", "FULL_CXX", "TAU_F90")
package_parts = {
"BFDINCLUDE":
("binutils_source",
lambda x: os.path.dirname(shlex.split(x)[0].lstrip("-I"))),
"UNWIND_INC": ("libunwind_source",
lambda x: os.path.dirname(x.lstrip("-I"))),
"PAPIDIR": ("papi_source", os.path.abspath),
"PDTDIR": ("pdt_source", os.path.abspath),
"SCOREPDIR": ("scorep_source", os.path.abspath)
}
tau_r = ''
for line in fin:
if line.startswith('#'):
continue
try:
key, val = [x.strip() for x in line.split('=', 1)]
except ValueError:
continue
if key == 'TAU_R':
tau_r = val.split()[0]
elif key in compiler_parts:
path = util.which(val.strip().split()[0].replace(
'$(TAU_R)', tau_r))
if not path:
self.logger.warning(
"Failed to parse %s in TAU Makefile '%s'", key,
makefile)
continue
matching_info = Knowledgebase.find_compiler(
os.path.basename(path))
if matching_info:
if len(matching_info) > 1:
self.logger.warning(
"Ambiguous compiler '%s' in TAU Makefile '%s'",
path, makefile)
comp = InstalledCompiler(path, matching_info[0])
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'",
attr.lower().replace("_", "-"),
comp.absolute_path)
while comp.wrapped:
comp = comp.wrapped
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'",
attr.lower().replace("_", "-"),
comp.absolute_path)
elif key in package_parts:
attr, operator = package_parts[key]
path = val.strip()
if not path:
path = None
else:
path = operator(path)
if not os.path.exists(path):
self.logger.warning(
"'%s' referenced by TAU Makefile '%s' doesn't exist",
path, makefile)
continue
setattr(args, attr, path)
self.logger.info(" --%s='%s'",
attr.replace("_source", ""), path)
def check_compiler(self, compiler_cmd, compiler_args):
"""Checks a compiler command its arguments for compatibility with this target configuration.
Checks that the given compiler matches at least one, **but possibly more**, of the compilers
used in the target. Also performs any special checkes for invalid compiler arguments,
e.g. -mmic is only for native KNC.
If the given compiler command and arguments are compatible with this target then information about
matching compiler installations is returned as a list of n :any:`InstalledCompiler` instances.
Args:
compiler_cmd (str): The compiler command as passed by the user.
compiler_args (list): Compiler command line arguments.
Returns:
list: Information about matching installed compilers as :any:`Compiler` instances.
Raises:
ConfigurationError: The compiler or command line arguments are incompatible with this target.
"""
if '-mmic' in compiler_args and self['host_arch'] != str(INTEL_KNC):
raise ConfigurationError(
"Host architecture of target '%s' is '%s'"
" but the '-mmic' compiler argument requires '%s'" %
(self['name'], self['host_arch'], INTEL_KNC),
"Select a different target",
"Create a new target with host architecture '%s'" % INTEL_KNC)
compiler_ctrl = Compiler.controller(self.storage)
absolute_path = util.which(compiler_cmd)
compiler_cmd = os.path.basename(compiler_cmd)
found = []
known_compilers = [comp for comp in self.compilers().itervalues()]
for info in Knowledgebase.find_compiler(command=compiler_cmd):
try:
compiler_record = self.populate(info.role.keyword)
except KeyError:
# Target was not configured with a compiler in this role
continue
compiler_path = compiler_record['path']
if (absolute_path and (compiler_path == absolute_path)
or (not absolute_path and
(os.path.basename(compiler_path) == compiler_cmd))):
found.append(compiler_record)
else:
# Target was configured with a wrapper compiler so check if that wrapper wraps this compiler
while 'wrapped' in compiler_record:
compiler_record = compiler_ctrl.one(
compiler_record['wrapped'])
known_compilers.append(compiler_record.installation())
compiler_path = compiler_record['path']
if (absolute_path and (compiler_path == absolute_path) or
(not absolute_path and
(os.path.basename(compiler_path) == compiler_cmd))):
found.append(compiler_record)
break
if not found:
parts = [
"No compiler in target '%s' matches '%s'." %
(self['name'], absolute_path or compiler_cmd),
"The known compiler commands are:"
]
parts.extend(' %s (%s)' %
(comp.absolute_path, comp.info.short_descr)
for comp in known_compilers)
hints = (
"Try one of the valid compiler commands",
"Create and select a new target configuration that uses the '%s' compiler"
% (absolute_path or compiler_cmd),
"Check loaded modules and the PATH environment variable")
raise ConfigurationError('\n'.join(parts), *hints)
return found
class CreateTest(tests.TestCase):
"""Tests for :any:`target.create`."""
def test_create(self):
self.reset_project_storage()
argv = ['targ02']
stdout, stderr = self.assertCommandReturnValue(0, create_cmd, argv)
self.assertIn(
'Added target \'targ02\' to project configuration \'proj1\'',
stdout)
self.assertFalse(stderr)
def test_no_args(self):
self.reset_project_storage()
_, stderr = self.assertNotCommandReturnValue(0, create_cmd, [])
self.assertIn('error: too few arguments', stderr)
def test_h_arg(self):
self.reset_project_storage()
stdout, _ = self.assertCommandReturnValue(0, create_cmd, ['-h'])
self.assertIn('Create target configurations.', stdout)
self.assertIn('Show this help message and exit', stdout)
def test_help_arg(self):
self.reset_project_storage()
stdout, _ = self.assertCommandReturnValue(0, create_cmd, ['--help'])
self.assertIn('Create target configurations.', stdout)
self.assertIn('Show this help message and exit', stdout)
def test_duplicatename(self):
self.reset_project_storage()
_, stderr = self.assertNotCommandReturnValue(0, create_cmd, ['targ1'])
self.assertIn('target create <target_name> [arguments]', stderr)
self.assertIn('target create: error: A target with name', stderr)
self.assertIn('already exists', stderr)
@tests.skipUnless(util.which('icc'),
"Intel compilers required for this test")
def test_host_family_intel(self):
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, create_cmd, ['test_targ', '--compilers', 'Intel'])
self.assertIn("Added target", stdout)
self.assertIn("test_targ", stdout)
self.assertFalse(stderr)
from taucmdr.cf.storage.levels import PROJECT_STORAGE
from taucmdr.model.target import Target
ctrl = Target.controller(PROJECT_STORAGE)
test_targ = ctrl.one({'name': 'test_targ'})
for role, expected in (CC, 'icc'), (CXX, 'icpc'), (FC, 'ifort'):
path = test_targ.populate(role.keyword)['path']
self.assertEqual(
os.path.basename(path), expected,
"Target[%s] is '%s', not '%s'" % (role, path, expected))
@tests.skipUnless(util.which('pgcc'),
"PGI compilers required for this test")
def test_host_family_pgi(self):
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, create_cmd, ['test_targ', '--compilers', 'PGI'])
self.assertIn("Added target", stdout)
self.assertIn("test_targ", stdout)
self.assertFalse(stderr)
from taucmdr.cf.storage.levels import PROJECT_STORAGE
from taucmdr.model.target import Target
ctrl = Target.controller(PROJECT_STORAGE)
test_targ = ctrl.one({'name': 'test_targ'})
path = test_targ.populate(CC.keyword)['path']
self.assertEqual('pgcc', os.path.basename(path),
"Target[CC] is '%s', not 'pgcc'" % path)
def test_cc_flag(self):
self.reset_project_storage()
cc_cmd = self.assertCompiler(CC)
stdout, stderr = self.assertCommandReturnValue(
0, create_cmd, ['test_targ', '--cc', cc_cmd])
self.assertIn("Added target 'test_targ' to project configuration",
stdout)
self.assertFalse(stderr)
def test_cxx_as_cc_flag(self):
self.reset_project_storage()
cxx_cmd = self.assertCompiler(CXX)
self.assertRaises(ConfigurationError, self.exec_command, create_cmd,
['test_targ', '--cc', cxx_cmd])
@tests.skipUnless(util.which('python'),
"Python 2 or 3 required for this test")
def test_python_init(self):
self.reset_project_storage(
['--python', 'T', '--python-interpreter', 'python'])
@tests.skipUnless(util.which('python2'), "Python 2 required for this test")
def test_python2_init(self):
self.reset_project_storage(
['--python', 'T', '--python-interpreter', 'python2'])
@tests.skipUnless(util.which('python3'), "Python 3 required for this test")
def test_python3_init(self):
self.reset_project_storage(
['--python', 'T', '--python-interpreter', 'python3'])
def _parse_tau_makefile(self, args):
# Parsing a TAU Makefile is a really hairy operation, so let's lift the limits
# pylint: disable=too-many-statements,too-many-locals
makefile = args.forced_makefile
if not util.path_accessible(makefile):
self.parser.error("Invalid TAU makefile: %s" % makefile)
tau_arch_name = os.path.basename(os.path.dirname(os.path.dirname(makefile)))
matches = [arch for arch in TauMagic.all() if arch.name == tau_arch_name]
if len(matches) == 1:
tau_arch = matches[0]
elif len(matches) == 0:
raise ConfigurationError("TAU Makefile '%s' targets an unrecognized TAU architecture: %s" %
(makefile, tau_arch_name))
else:
for arch in matches:
if arch.architecture == HOST_ARCH and arch.operating_system == HOST_OS:
tau_arch = arch
break
else:
parts = ["TAU Makefile '%s' targets an ambiguous TAU architecture: %s" % (makefile, tau_arch_name),
"It could be any of these:"]
parts.extend([" - %s on %s" % (arch.operating_system.name, arch.architecture.name)
for arch in matches])
raise ConfigurationError("\n".join(parts))
self.logger.info("Parsing TAU Makefile '%s' to populate command line arguments:", makefile)
args.host_arch = tau_arch.architecture.name
self.logger.info(" --host-arch='%s'", args.host_arch)
args.host_os = tau_arch.operating_system.name
self.logger.info(" --host-os='%s'", args.host_os)
args.tau_source = os.path.abspath(os.path.join(os.path.dirname(makefile), '..', '..'))
self.logger.info(" --taucmdr='%s'", args.tau_source)
with open(makefile, 'r') as fin:
compiler_parts = ("FULL_CC", "FULL_CXX", "TAU_F90")
package_parts = {"BFDINCLUDE": ("binutils_source", lambda x: os.path.dirname(x.lstrip("-I"))),
"UNWIND_INC": ("libunwind_source", lambda x: os.path.dirname(x.lstrip("-I"))),
"PAPIDIR": ("papi_source", os.path.abspath),
"PDTDIR": ("pdt_source", os.path.abspath),
"SCOREPDIR": ("scorep_source", os.path.abspath)}
tau_r = ''
for line in fin:
if line.startswith('#'):
continue
try:
key, val = [x.strip() for x in line.split('=', 1)]
except ValueError:
continue
if key == 'TAU_R':
tau_r = val.split()[0]
elif key in compiler_parts:
path = util.which(val.strip().split()[0].replace('$(TAU_R)', tau_r))
if not path:
self.logger.warning("Failed to parse %s in TAU Makefile '%s'", key, makefile)
continue
matching_info = Knowledgebase.find_compiler(os.path.basename(path))
if matching_info:
if len(matching_info) > 1:
self.logger.warning("Ambiguous compiler '%s' in TAU Makefile '%s'", path, makefile)
comp = InstalledCompiler(path, matching_info[0])
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path)
while comp.wrapped:
comp = comp.wrapped
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path)
elif key in package_parts:
attr, operator = package_parts[key]
path = val.strip()
if not path:
path = None
else:
path = operator(path)
if not os.path.exists(path):
self.logger.warning("'%s' referenced by TAU Makefile '%s' doesn't exist", path, makefile)
continue
setattr(args, attr, path)
self.logger.info(" --%s='%s'", attr.replace("_source", ""), path)
class ExportTest(tests.TestCase):
@tests.skipUnless(util.which('java'),
"A java interpreter is required for this test")
def test_export_tau_profile(self):
self.reset_project_storage(['--profile', 'tau', '--trace', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['profile'], 'tau')
self.assertEqual(meas['trace'], 'none')
self.assertManagedBuild(0, CC, [], 'hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd,
['./a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.ppk'
self.assertTrue(os.path.exists(export_file))
@tests.skipUnlessHaveCompiler(MPI_CC)
def test_export_merged_profile(self):
self.reset_project_storage(
['--mpi', '--profile', 'merged', '--trace', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['profile'], 'merged')
self.assertEqual(meas['trace'], 'none')
self.assertManagedBuild(0, MPI_CC, [], 'mpi_hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd,
['mpirun', '-np', '4', './a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.xml.gz'
self.assertTrue(os.path.exists(export_file))
@tests.skipUnlessHaveCompiler(MPI_CC)
def test_export_cubex(self):
self.reset_project_storage(
['--mpi', '--profile', 'cubex', '--trace', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['profile'], 'cubex')
self.assertEqual(meas['trace'], 'none')
self.assertManagedBuild(0, MPI_CC, [], 'mpi_hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd,
['mpirun', '-np', '4', './a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.cubex'
self.assertTrue(os.path.exists(export_file))
@tests.skipUnless(util.which('java'),
"A java interpreter is required for this test")
def test_export_slog2(self):
self.reset_project_storage(['--trace', 'slog2', '--profile', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['trace'], 'slog2')
self.assertEqual(meas['profile'], 'none')
self.assertManagedBuild(0, CC, [], 'hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd,
['./a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.slog2'
self.assertTrue(os.path.exists(export_file))
@tests.skipUnlessHaveCompiler(MPI_CC)
def test_export_otf2(self):
self.reset_project_storage(
['--mpi', '--trace', 'otf2', '--profile', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['trace'], 'otf2')
self.assertEqual(meas['profile'], 'none')
self.assertManagedBuild(0, MPI_CC, [], 'mpi_hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd,
['mpirun', '-np', '4', './a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.tgz'
self.assertTrue(os.path.exists(export_file))
