def make_install(self, flags):
"""Invoke `make install`.
Args:
flags (list): Command line flags to pass to `make`.
Raises:
SoftwarePackageError: Configuration failed.
"""
assert self._src_prefix
LOGGER.debug("Installing %s to '%s'", self.name, self.install_prefix)
cmd = ['make', 'install'] + parallel_make_flags() + flags
LOGGER.info("Installing %s...", self.title)
if util.create_subprocess(cmd,
cwd=self._src_prefix,
stdout=False,
show_progress=True):
cmd = ['make', 'install'] + flags
if util.create_subprocess(cmd,
cwd=self._src_prefix,
stdout=False,
show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('%s installation failed' %
self.title)
# Some systems use lib64 instead of lib
if os.path.isdir(self.lib_path +
'64') and not os.path.isdir(self.lib_path):
os.symlink(self.lib_path + '64', self.lib_path)
def make(self, flags):
"""Invoke `make`.
Args:
flags (list): Command line flags to pass to `make`.
Raises:
SoftwarePackageError: Compilation failed.
"""
assert self._src_prefix
LOGGER.debug("Making %s at '%s'", self.name, self._src_prefix)
cmd = ['make'] + parallel_make_flags() + flags
LOGGER.info("Compiling %s...", self.title)
if util.create_subprocess(cmd,
cwd=self._src_prefix,
stdout=False,
show_progress=True):
cmd = ['make'] + flags
if util.create_subprocess(cmd,
cwd=self._src_prefix,
stdout=False,
show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('%s compilation failed' %
self.title)
def queue_command(self, expr, cmd, cwd, env):
"""Execute a command as part of an experiment trial.
Creates a new subprocess for the command and checks for TAU data files
when the subprocess exits.
Args:
expr (Experiment): Experiment data.
cmd (str): Command to profile, with command line arguments.
cwd (str): Working directory to perform trial in.
env (dict): Environment variables to set before performing the trial.
Returns:
int: Subprocess return code.
"""
cmd_str = ' '.join(cmd)
tau_env_opts = sorted('%s=%s' % (key, val) for key, val in env.iteritems()
if (key.startswith('TAU_') or
key.startswith('SCOREP_') or
key in ('PROFILEDIR', 'TRACEDIR')))
LOGGER.info('\n'.join(tau_env_opts))
LOGGER.info(cmd_str)
try:
retval = util.create_subprocess(cmd, cwd=cwd, env=env, log=False)
except OSError as err:
target = expr.populate('target')
errno_hint = {errno.EPERM: "Check filesystem permissions",
errno.ENOENT: "Check paths and command line arguments",
errno.ENOEXEC: "Check that this host supports '%s'" % target['host_arch']}
raise TrialError("Couldn't execute %s: %s" % (cmd_str, err), errno_hint.get(err.errno, None))
if retval:
LOGGER.warning("Return code %d from '%s'", retval, cmd_str)
return retval
def configure(self, _):
family_flags = {GNU.name: '-GNU', INTEL.name: '-icpc', PGI.name: '-pgCC'}
compiler_flag = family_flags.get(self.compilers[CXX].info.family.name, '')
cmd = ['./configure', '-prefix=' + self.install_prefix, compiler_flag]
LOGGER.info("Configuring PDT...")
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
raise SoftwarePackageError('%s configure failed' % self.title)
def make(self, flags):
"""Invoke `make`.
Args:
flags (list): Command line flags to pass to `make`.
Raises:
SoftwarePackageError: Compilation failed.
"""
assert self._src_prefix
LOGGER.debug("Making %s at '%s'", self.name, self._src_prefix)
cmd = ['make'] + parallel_make_flags() + flags
LOGGER.info("Compiling %s...", self.title)
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
cmd = ['make'] + flags
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('%s compilation failed' % self.title)
def _configure_edg4x_rose(self):
LOGGER.info('edg4x-rose parser configuration failed. Retrying...')
cwd = os.path.join(self.install_prefix, 'contrib', 'rose', 'edg44', self.tau_magic.name, 'roseparse')
if not os.path.exists(cwd):
LOGGER.info("roseparse not available on %s. Good luck!", self.tau_magic.name)
return
if util.create_subprocess(['./configure'], cwd=cwd, stdout=False, show_progress=True):
raise SoftwarePackageError('Unable to configure edg4x-rose parsers')
LOGGER.info("'edg4x-rose parser configuration successful. Continuing %s verification...", self.title)
def make_install(self, flags):
"""Invoke `make install`.
Args:
flags (list): Command line flags to pass to `make`.
Raises:
SoftwarePackageError: Configuration failed.
"""
assert self._src_prefix
LOGGER.debug("Installing %s to '%s'", self.name, self.install_prefix)
cmd = ['make', 'install'] + parallel_make_flags() + flags
LOGGER.info("Installing %s...", self.title)
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
cmd = ['make', 'install'] + flags
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('%s installation failed' % self.title)
# Some systems use lib64 instead of lib
if os.path.isdir(self.lib_path+'64') and not os.path.isdir(self.lib_path):
os.symlink(self.lib_path+'64', self.lib_path)
def cmake(self, flags):
"""Invoke `cmake`.
Args:
flags (list): Command line flags to pass to `cmake`.
Raises:
SoftwarePackageError: Configuration failed.
"""
assert self._src_prefix
cmake = self._get_cmake()
cmd = [cmake, '-DCMAKE_INSTALL_PREFIX=%s' % self.install_prefix] + flags
LOGGER.info("Executing CMake for %s...", self.title)
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('CMake failed for %s' %self.title)
def configure(self, flags):
"""Invoke `configure`.
Args:
flags (list): Command line flags to pass to `configure`.
Raises:
SoftwarePackageError: Configuration failed.
"""
assert self._src_prefix
LOGGER.debug("Configuring %s at '%s'", self.name, self._src_prefix)
cmd = ['./configure', '--prefix=%s' % self.install_prefix] + flags
LOGGER.info("Configuring %s...", self.title)
if util.create_subprocess(cmd, cwd=self._src_prefix, stdout=False, show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('%s configure failed' % self.title)
def configure(self, flags):
"""Invoke `configure`.
Args:
flags (list): Command line flags to pass to `configure`.
Raises:
SoftwarePackageError: Configuration failed.
"""
assert self._src_prefix
LOGGER.debug("Configuring %s at '%s'", self.name, self._src_prefix)
cmd = ['./configure', '--prefix=%s' % self.install_prefix] + flags
LOGGER.info("Configuring %s...", self.title)
if util.create_subprocess(cmd,
cwd=self._src_prefix,
stdout=False,
show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('%s configure failed' % self.title)
def cmake(self, flags):
"""Invoke `cmake`.
Args:
flags (list): Command line flags to pass to `cmake`.
Raises:
SoftwarePackageError: Configuration failed.
"""
assert self._src_prefix
cmake = self._get_cmake()
cmd = [cmake, '-DCMAKE_INSTALL_PREFIX=%s' % self.install_prefix
] + flags
LOGGER.info("Executing CMake for %s...", self.title)
if util.create_subprocess(cmd,
cwd=self._src_prefix,
stdout=False,
show_progress=True):
util.add_error_stack(self._src_prefix)
raise SoftwarePackageError('CMake failed for %s' % self.title)
def execute_command(self, expr, cmd, cwd, env, record_output=False):
"""Execute a command as part of an experiment trial.
Creates a new subprocess for the command and checks for TAU data files
when the subprocess exits.
Args:
expr (Experiment): Experiment data.
cmd (str): Command to profile, with command line arguments.
cwd (str): Working directory to perform trial in.
env (dict): Environment variables to set before performing the trial.
Returns:
int: Subprocess return code.
"""
cmd_str = ' '.join(cmd)
tau_env_opts = sorted(
'%s=%s' % (key, val) for key, val in env.iteritems()
if (key.startswith('TAU_') or key.startswith('SCOREP_') or key in (
'PROFILEDIR', 'TRACEDIR')))
LOGGER.info('\n'.join(tau_env_opts))
LOGGER.info(cmd_str)
try:
begin_time = time.time()
ret = util.create_subprocess(cmd,
cwd=cwd,
env=env,
log=False,
record_output=record_output)
elapsed = time.time() - begin_time
if record_output:
retval = ret[0]
output = base64.b64encode(repr(ret[1]))
else:
retval = ret
except OSError as err:
target = expr.populate('target')
errno_hint = {
errno.EPERM:
"Check filesystem permissions",
errno.ENOENT:
"Check paths and command line arguments",
errno.ENOEXEC:
"Check that this host supports '%s'" % target['host_arch']
}
raise TrialError("Couldn't execute %s: %s" % (cmd_str, err),
errno_hint.get(err.errno, None))
measurement = expr.populate('measurement')
profiles = []
for pat in 'profile.*.*.*', 'MULTI__*/profile.*.*.*', 'tauprofile.xml', '*.cubex':
profiles.extend(glob.glob(os.path.join(self.prefix, pat)))
if profiles:
LOGGER.info("Trial %s produced %s profile files.", self['number'],
len(profiles))
negative_profiles = [
prof for prof in profiles if 'profile.-1' in prof
]
if negative_profiles:
LOGGER.warning(
"Trial %s produced a profile with negative node number!"
" This usually indicates that process-level parallelism was not initialized,"
" (e.g. MPI_Init() was not called) or there was a problem in instrumentation."
" Check the compilation output and verify that MPI_Init (or similar) was called.",
self['number'])
for fname in negative_profiles:
new_name = fname.replace(".-1.", ".0.")
if not os.path.exists(new_name):
LOGGER.info("Renaming %s to %s", fname, new_name)
os.rename(fname, new_name)
else:
raise ConfigurationError(
"The profile numbers for trial %d cannot be corrected.",
"Check that the application configuration is correct.",
"Check that the measurement configuration is correct.",
"Check for instrumentation failure in the compilation log."
)
elif measurement['profile'] != 'none':
raise TrialError("Trial did not produce any profiles.")
traces = []
for pat in '*.slog2', '*.trc', '*.edf', 'traces/*.def', 'traces/*.evt', 'traces.otf2':
traces.extend(glob.glob(os.path.join(self.prefix, pat)))
if traces:
LOGGER.info("Trial %s produced %s trace files.", self['number'],
len(traces))
elif measurement['trace'] != 'none':
raise TrialError(
"Application completed successfuly but did not produce any traces."
)
if retval:
LOGGER.warning("Return code %d from '%s'", retval, cmd_str)
if record_output:
return retval, output, elapsed
else:
return retval, elapsed
def execute_command(self, expr, cmd, cwd, env):
"""Execute a command as part of an experiment trial.
Creates a new subprocess for the command and checks for TAU data files
when the subprocess exits.
Args:
expr (Experiment): Experiment data.
cmd (str): Command to profile, with command line arguments.
cwd (str): Working directory to perform trial in.
env (dict): Environment variables to set before performing the trial.
Returns:
int: Subprocess return code.
"""
cmd_str = ' '.join(cmd)
tau_env_opts = sorted('%s=%s' % (key, val) for key, val in env.iteritems()
if (key.startswith('TAU_') or
key.startswith('SCOREP_') or
key in ('PROFILEDIR', 'TRACEDIR')))
LOGGER.info('\n'.join(tau_env_opts))
LOGGER.info(cmd_str)
try:
begin_time = time.time()
retval = util.create_subprocess(cmd, cwd=cwd, env=env, log=False)
elapsed = time.time() - begin_time
except OSError as err:
target = expr.populate('target')
errno_hint = {errno.EPERM: "Check filesystem permissions",
errno.ENOENT: "Check paths and command line arguments",
errno.ENOEXEC: "Check that this host supports '%s'" % target['host_arch']}
raise TrialError("Couldn't execute %s: %s" % (cmd_str, err), errno_hint.get(err.errno, None))
measurement = expr.populate('measurement')
profiles = []
for pat in 'profile.*.*.*', 'MULTI__*/profile.*.*.*', 'tauprofile.xml', '*.cubex':
profiles.extend(glob.glob(os.path.join(self.prefix, pat)))
if profiles:
LOGGER.info("Trial %s produced %s profile files.", self['number'], len(profiles))
negative_profiles = [prof for prof in profiles if 'profile.-1' in prof]
if negative_profiles:
LOGGER.warning("Trial %s produced a profile with negative node number!"
" This usually indicates that process-level parallelism was not initialized,"
" (e.g. MPI_Init() was not called) or there was a problem in instrumentation."
" Check the compilation output and verify that MPI_Init (or similar) was called.",
self['number'])
for fname in negative_profiles:
new_name = fname.replace(".-1.", ".0.")
if not os.path.exists(new_name):
LOGGER.info("Renaming %s to %s", fname, new_name)
os.rename(fname, new_name)
else:
raise ConfigurationError("The profile numbers for trial %d cannot be corrected.",
"Check that the application configuration is correct.",
"Check that the measurement configuration is correct.",
"Check for instrumentation failure in the compilation log.")
elif measurement['profile'] != 'none':
raise TrialError("Trial did not produce any profiles.")
traces = []
for pat in '*.slog2', '*.trc', '*.edf', 'traces/*.def', 'traces/*.evt', 'traces.otf2':
traces.extend(glob.glob(os.path.join(self.prefix, pat)))
if traces:
LOGGER.info("Trial %s produced %s trace files.", self['number'], len(traces))
elif measurement['trace'] != 'none':
raise TrialError("Application completed successfuly but did not produce any traces.")
if retval:
LOGGER.warning("Return code %d from '%s'", retval, cmd_str)
return retval, elapsed
def _configure_edg4x_rose(self):
LOGGER.info('edg4x-rose parser configuration failed. Retrying...')
cwd = os.path.join(self.install_prefix, 'contrib', 'rose', 'edg44', self.tau_magic.name, 'roseparse')
if util.create_subprocess(['./configure'], cwd=cwd, stdout=False, show_progress=True):
raise SoftwarePackageError('Unable to configure edg4x-rose parsers')
LOGGER.info("'edg4x-rose parser configuration successful. Continuing %s verification...", self.title)
