def _get_install_prefix(self):
if not self._install_prefix:
if self.unmanaged:
self._set_install_prefix(self.src)
else:
tag = self._get_install_tag()
# Search the storage hierarchy for an existing installation
for storage in reversed(ORDERED_LEVELS):
try:
self._set_install_prefix(
os.path.join(storage.prefix, self.name, tag))
self.verify()
except (StorageError, SoftwarePackageError) as err:
LOGGER.debug(err)
continue
else:
break
else:
# No existing installation found, install at highest writable storage level
self._set_install_prefix(
os.path.join(highest_writable_storage().prefix,
self.name, tag))
LOGGER.debug("%s installation prefix is %s", self.name,
self._install_prefix)
return self._install_prefix
def _acquire_source(self, reuse_archive):
archive_file = os.path.basename(self.src)
if reuse_archive:
for storage in ORDERED_LEVELS:
try:
archive = os.path.join(storage.prefix, "src", archive_file)
except StorageError:
continue
if os.path.exists(archive):
return archive
archive_prefix = os.path.join(highest_writable_storage().prefix, "src")
archive = os.path.join(archive_prefix, os.path.basename(self.src))
try:
util.download(self.src, archive)
except IOError:
hints = ("If a firewall is blocking access to this server, use another method to download "
"'%s' and copy that file to '%s' before trying this operation." % (self.src, archive_prefix),
"Check that the file or directory is accessible")
raise ConfigurationError("Cannot acquire source archive '%s'." % self.src, *hints)
return archive
def _get_install_prefix(self):
if not self._install_prefix:
if self.unmanaged:
self._set_install_prefix(self.src)
else:
tag = self._get_install_tag()
# Search the storage hierarchy for an existing installation
for storage in reversed(ORDERED_LEVELS):
try:
self._set_install_prefix(os.path.join(storage.prefix, self.name, tag))
self.verify()
except (StorageError, SoftwarePackageError) as err:
LOGGER.debug(err)
continue
else:
break
else:
# No existing installation found, install at highest writable storage level
self._set_install_prefix(os.path.join(highest_writable_storage().prefix, self.name, tag))
LOGGER.debug("%s installation prefix is %s", self.name, self._install_prefix)
return self._install_prefix
def tmpfs_prefix():
"""Path to a uniquely named directory in a temporary filesystem, ideally a ramdisk.
/dev/shm is the preferred tmpfs, but if it's unavailable or mounted with noexec then
fall back to tempfile.gettemdir(), which is usually /tmp. If that filesystem is also
unavailable then use the filesystem prefix of the highest writable storage container.
Returns:
str: Path to a uniquely-named directory in the temporary filesystem. The directory
and all its contents **will be deleted** when the program exits if it installs
correctly.
"""
try:
tmp_prefix = tmpfs_prefix.value
except AttributeError:
import tempfile
import subprocess
from stat import S_IRUSR, S_IWUSR, S_IEXEC
for prefix in "/dev/shm", tempfile.gettempdir(), highest_writable_storage().prefix:
try:
tmp_prefix = util.mkdtemp(dir=prefix)
except (OSError, IOError) as err:
LOGGER.debug(err)
continue
# Check execute privilages some distros mount tmpfs with the noexec option.
try:
with tempfile.NamedTemporaryFile(dir=tmp_prefix, delete=False) as tmp_file:
tmp_path = tmp_file.name
tmp_file.write("#!/bin/sh\nexit 0")
os.chmod(tmp_path, S_IRUSR | S_IWUSR | S_IEXEC)
subprocess.check_call([tmp_path])
except (OSError, IOError, subprocess.CalledProcessError) as err:
LOGGER.debug(err)
continue
else:
break
tmpfs_prefix.value = tmp_prefix
return tmp_prefix
def acquire_source(self, reuse_archive=True):
"""Acquires package source code archive file via download or file copy.
If the package is configured to use an existing installation as the source then
this routine does nothing.
Args:
reuse_archive (bool): If True don't download, just confirm that the archive exists.
Returns:
str: Absolute path to the source archive.
Raises:
ConfigurationError: Package source code not provided or couldn't be acquired.
"""
if not self.src:
raise ConfigurationError("No source code provided for %s" % self.title)
if self.unmanaged:
return self.src
archive_file = os.path.basename(self.src)
if reuse_archive:
for storage in reversed(ORDERED_LEVELS):
try:
archive = os.path.join(storage.prefix, "src", archive_file)
except StorageError:
continue
if os.path.exists(archive):
return archive
archive_prefix = os.path.join(highest_writable_storage().prefix, "src")
archive = os.path.join(archive_prefix, os.path.basename(self.src))
try:
util.download(self.src, archive)
except IOError:
hints = ("If a firewall is blocking access to this server, use another method to download "
"'%s' and copy that file to '%s' before trying this operation." % (self.src, archive_prefix),
"Check that the file or directory is accessible")
raise ConfigurationError("Cannot acquire source archive '%s'." % self.src, *hints)
return archive
def _acquire_source(self, reuse_archive):
archive_file = os.path.basename(self.src)
if reuse_archive:
for storage in ORDERED_LEVELS:
try:
archive = os.path.join(storage.prefix, "src", archive_file)
except StorageError:
continue
if os.path.exists(archive):
return archive
archive_prefix = os.path.join(highest_writable_storage().prefix, "src")
archive = os.path.join(archive_prefix, os.path.basename(self.src))
try:
util.download(self.src, archive)
except IOError:
hints = (
"If a firewall is blocking access to this server, use another method to download "
"'%s' and copy that file to '%s' before trying this operation."
% (self.src, archive_prefix),
"Check that the file or directory is accessible")
raise ConfigurationError(
"Cannot acquire source archive '%s'." % self.src, *hints)
return archive
class Experiment(Model):
"""Experiment data model."""
__attributes__ = attributes
__controller__ = ExperimentController
@classmethod
def controller(cls, storage=PROJECT_STORAGE):
return cls.__controller__(cls, storage)
@classmethod
def select(cls, name):
"""Changes the selected experiment in the current project.
Raises:
ExperimentSelectionError: No experiment with the given name in the currently selected project.
Args:
name (str): Name of the experiment to select.
"""
proj_ctrl = Project.controller()
proj = proj_ctrl.selected()
expr_ctrl = cls.controller()
data = {"name": name, "project": proj.eid}
matching = expr_ctrl.search(data)
if not matching:
raise ExperimentSelectionError(
"There is no experiment named '%s' in project '%s'." %
(name, proj['name']))
elif len(matching) > 1:
raise InternalError(
"More than one experiment with data %r exists!" % data)
else:
expr = matching[0]
proj_ctrl.select(proj, expr)
@classmethod
def rebuild_required(cls):
"""Builds a string indicating if an application rebuild is required.
Rebuild information is taken from the 'rebuild_required' topic.
Returns:
str: String indicating why an application rebuild is required.
"""
def _fmt(val):
if isinstance(val, list):
return "[%s]" % ", ".join(val)
elif isinstance(val, basestring):
return "'%s'" % val
else:
return str(val)
rebuild_required = cls.controller().pop_topic('rebuild_required')
if not rebuild_required:
return ''
parts = ["Application rebuild required:"]
for changed in rebuild_required:
for attr, change in changed.iteritems():
old, new = (_fmt(x) for x in change)
if old is None:
parts.append(" - %s is now set to %s" % (attr, new))
elif new is None:
parts.append(" - %s is now unset" % attr)
else:
parts.append(" - %s changed from %s to %s" %
(attr, old, new))
return '\n'.join(parts)
@property
def prefix(self):
with fasteners.InterProcessLock(
os.path.join(PROJECT_STORAGE.prefix, '.lock')):
return os.path.join(self.populate('project').prefix, self['name'])
def verify(self):
"""Checks all components of the experiment for mutual compatibility."""
populated = self.populate()
proj = populated['project']
targ = populated['target']
app = populated['application']
meas = populated['measurement']
for model in targ, app, meas:
if proj.eid not in model['projects']:
raise IncompatibleRecordError(
"%s '%s' is not a member of project configuration '%s'." %
(model.name, model['name'], proj['name']))
for lhs in [targ, app, meas]:
for rhs in [targ, app, meas]:
lhs.check_compatibility(rhs)
def on_create(self):
self.verify()
try:
util.mkdirp(self.prefix)
except:
raise ConfigurationError(
'Cannot create directory %r' % self.prefix,
'Check that you have `write` access')
def on_delete(self):
try:
util.rmtree(self.prefix)
except Exception as err: # pylint: disable=broad-except
if os.path.exists(self.prefix):
LOGGER.error("Could not remove experiment data at '%s': %s",
self.prefix, err)
def data_size(self):
return sum([
int(trial.get('data_size', 0)) for trial in self.populate('trials')
])
def next_trial_number(self):
trials = self.populate(attribute='trials', defaults=True)
for i, j in enumerate(sorted([trial['number'] for trial in trials])):
if i != j:
return i
return len(trials)
@fasteners.interprocess_locked(
os.path.join(highest_writable_storage().prefix, '.lock'))
def configure(self):
"""Sets up the Experiment for a new trial.
Installs or configures TAU and all its dependencies. After calling this
function, the experiment is ready to operate on the user's application.
Returns:
TauInstallation: Object handle for the TAU installation.
"""
from taucmdr.cf.software.tau_installation import TauInstallation
LOGGER.debug("Configuring experiment %s", self['name'])
with fasteners.InterProcessLock(
os.path.join(PROJECT_STORAGE.prefix, '.lock')):
populated = self.populate(defaults=True)
target = populated['target']
application = populated['application']
measurement = populated['measurement']
baseline = measurement.get_or_default('baseline')
tau = TauInstallation(\
target.sources(),
target_arch=target.architecture(),
target_os=target.operating_system(),
compilers=target.compilers(),
# Use a minimal configuration for the baseline measurement
minimal=baseline,
# TAU feature suppport
application_linkage=application.get_or_default('linkage'),
openmp_support=application.get_or_default('openmp'),
pthreads_support=application.get_or_default('pthreads'),
tbb_support=application.get_or_default('tbb'),
mpi_support=application.get_or_default('mpi'),
mpi_libraries=target.get('mpi_libraries', []),
cuda_support=application.get_or_default('cuda'),
cuda_prefix=target.get('cuda_toolkit', None),
opencl_support=application.get_or_default('opencl'),
opencl_prefix=target.get('opencl', None),
shmem_support=application.get_or_default('shmem'),
shmem_libraries=target.get('shmem_libraries', []),
mpc_support=application.get_or_default('mpc'),
# Instrumentation methods and options
source_inst=measurement.get_or_default('source_inst'),
compiler_inst=measurement.get_or_default('compiler_inst'),
keep_inst_files=measurement.get_or_default('keep_inst_files'),
reuse_inst_files=measurement.get_or_default('reuse_inst_files'),
select_file=application.get('select_file', None),
# Measurement methods and options
baseline=baseline,
profile=measurement.get_or_default('profile'),
trace=measurement.get_or_default('trace'),
sample=measurement.get_or_default('sample'),
metrics=measurement.get_or_default('metrics'),
measure_io=measurement.get_or_default('io'),
measure_mpi=measurement.get_or_default('mpi'),
measure_openmp=measurement.get_or_default('openmp'),
measure_opencl=measurement.get_or_default('opencl'),
measure_cuda=measurement.get_or_default('cuda'),
measure_shmem=measurement.get_or_default('shmem'),
measure_heap_usage=measurement.get_or_default('heap_usage'),
measure_system_load=measurement.get_or_default('system_load'),
measure_memory_alloc=measurement.get_or_default('memory_alloc'),
measure_comm_matrix=measurement.get_or_default('comm_matrix'),
measure_callsite=measurement.get_or_default('callsite'),
callpath_depth=measurement.get_or_default('callpath'),
throttle=measurement.get_or_default('throttle'),
metadata_merge=measurement.get_or_default('metadata_merge'),
throttle_per_call=measurement.get_or_default('throttle_per_call'),
throttle_num_calls=measurement.get_or_default('throttle_num_calls'),
forced_makefile=target.get('forced_makefile', None))
tau.install()
if not baseline:
self.controller(self.storage).update(
{'tau_makefile': os.path.basename(tau.get_makefile())},
self.eid)
return tau
def managed_build(self, compiler_cmd, compiler_args):
"""Uses this experiment to perform a build operation.
Checks that this experiment is compatible with the desired build operation,
prepares the experiment, and performs the operation.
Args:
compiler_cmd (str): The compiler command intercepted by TAU Commander.
compiler_args (list): Compiler command line arguments intercepted by TAU Commander.
Raises:
ConfigurationError: The experiment is not configured to perform the desired build.
Returns:
int: Build subprocess return code.
"""
LOGGER.debug("Managed build: %s", [compiler_cmd] + compiler_args)
target = self.populate('target')
application = self.populate('application')
target_compilers = target.check_compiler(compiler_cmd, compiler_args)
try:
found_compiler = application.check_compiler(target_compilers)
except ConfigurationError as err:
msg = err.value + (
"\nTAU will add additional compiler options "
"and attempt to continue but this may have unexpected results."
)
LOGGER.warning(msg)
found_compiler = target_compilers[0]
# We've found a candidate compiler. Check that this compiler record is still valid.
installed_compiler = found_compiler.verify()
tau = self.configure()
meas = self.populate('measurement')
try:
tau.force_tau_options = meas['force_tau_options']
except KeyError:
pass
else:
LOGGER.warning("Measurement '%s' forces TAU_OPTIONS='%s'",
meas['name'], ' '.join(tau.force_tau_options))
return tau.compile(installed_compiler, compiler_args)
def managed_run(self, launcher_cmd, application_cmds, description=None):
"""Uses this experiment to run an application command.
Performs all relevent system preparation tasks to run the user's application
under the specified experimental configuration.
Args:
launcher_cmd (list): Application launcher with command line arguments.
application_cmds (list): List of application executables with command line arguments (list of lists).
description (str): If not None, a description of the run.
Raises:
ConfigurationError: The experiment is not configured to perform the desired run.
Returns:
int: Application subprocess return code.
"""
tau = self.configure()
application = self.populate('application')
for application_cmd in application_cmds:
cmd0 = application_cmd[0]
linkage = util.get_binary_linkage(cmd0)
if linkage is None:
LOGGER.warning("Unable to check application linkage on '%s'",
cmd0)
break
if linkage != application['linkage']:
LOGGER.warning(
"Application configuration %s specifies %s linkage but '%s' has %s linkage",
application['name'], application['linkage'], cmd0, linkage)
cmd, env = tau.get_application_command(launcher_cmd, application_cmds)
proj = self.populate('project')
return Trial.controller(self.storage).perform(proj, cmd, os.getcwd(),
env, description)
def trials(self, trial_numbers=None):
"""Get a list of modeled trial records.
If `bool(trial_numbers)` is False, return the most recent trial.
Otherwise return a list of Trial objects for the given trial numbers.
Args:
trial_numbers (list): List of numbers of trials to retrieve.
Returns:
list: Modeled trial records.
Raises:
ConfigurationError: Invalid trial number or no trials in selected experiment.
"""
if trial_numbers:
for num in trial_numbers:
trials = []
found = Trial.controller(self.storage).one({
'experiment': self.eid,
'number': num
})
if not found:
raise ConfigurationError(
"Experiment '%s' has no trial with number %s" %
(self.name, num))
trials.append(found)
return trials
else:
trials = self.populate('trials')
if not trials:
raise ConfigurationError("No trials in experiment %s" %
self['name'])
found = trials[0]
for trial in trials[1:]:
if trial['begin_time'] > found['begin_time']:
found = trial
return [found]
def tmpfs_prefix():
"""Path to a uniquely named directory in a temporary filesystem, ideally a ramdisk.
/dev/shm is the preferred tmpfs, but if it's unavailable or mounted with noexec then
fall back to tempfile.gettemdir(), which is usually /tmp. If that filesystem is also
unavailable then use the filesystem prefix of the highest writable storage container.
An attempt is made to ensure that there is at least 2GiB of free space in the
selected filesystem. If
Returns:
str: Path to a uniquely-named directory in the temporary filesystem. The directory
and all its contents **will be deleted** when the program exits if it installs
correctly.
"""
try:
return tmpfs_prefix.value
except AttributeError:
import tempfile
import subprocess
from stat import S_IRUSR, S_IWUSR, S_IEXEC
candidate = None
for prefix in "/dev/shm", tempfile.gettempdir(
), highest_writable_storage().prefix:
try:
tmp_prefix = util.mkdtemp(dir=prefix)
except (OSError, IOError) as err:
LOGGER.debug(err)
continue
# Check execute privilages some distros mount tmpfs with the noexec option.
check_exe_script = None
try:
with tempfile.NamedTemporaryFile(dir=tmp_prefix,
delete=False) as tmp_file:
check_exe_script = tmp_file.name
tmp_file.write("#!/bin/sh\nexit 0")
os.chmod(check_exe_script, S_IRUSR | S_IWUSR | S_IEXEC)
subprocess.check_call([check_exe_script])
except (OSError, IOError, subprocess.CalledProcessError) as err:
LOGGER.debug(err)
continue
try:
statvfs = os.statvfs(check_exe_script)
except (OSError, IOError) as err:
LOGGER.debug(err)
if candidate is None:
candidate = tmp_prefix
continue
else:
free_mib = (statvfs.f_frsize * statvfs.f_bavail) / 0x100000
LOGGER.debug("%s: %sMB free", tmp_prefix, free_mib)
if free_mib < 2000:
continue
if check_exe_script:
os.remove(check_exe_script)
break
else:
if not candidate:
raise ConfigurationError(
"No filesystem has at least 2GB free space and supports executables."
)
tmp_prefix = candidate
LOGGER.warning("Unable to count available bytes in '%s'",
tmp_prefix)
tmpfs_prefix.value = tmp_prefix
LOGGER.debug("Temporary prefix: '%s'", tmp_prefix)
return tmp_prefix
def tmpfs_prefix():
"""Path to a uniquely named directory in a temporary filesystem, ideally a ramdisk.
/dev/shm is the preferred tmpfs, but if it's unavailable or mounted with noexec then
fall back to tempfile.gettemdir(), which is usually /tmp. If that filesystem is also
unavailable then use the filesystem prefix of the highest writable storage container.
An attempt is made to ensure that there is at least 2GiB of free space in the
selected filesystem. If
Returns:
str: Path to a uniquely-named directory in the temporary filesystem. The directory
and all its contents **will be deleted** when the program exits if it installs
correctly.
"""
try:
return tmpfs_prefix.value
except AttributeError:
import tempfile
import subprocess
from stat import S_IRUSR, S_IWUSR, S_IEXEC
candidate = None
for prefix in "/dev/shm", tempfile.gettempdir(), highest_writable_storage().prefix:
try:
tmp_prefix = util.mkdtemp(dir=prefix)
except (OSError, IOError) as err:
LOGGER.debug(err)
continue
# Check execute privilages some distros mount tmpfs with the noexec option.
check_exe_script = None
try:
with tempfile.NamedTemporaryFile(dir=tmp_prefix, delete=False) as tmp_file:
check_exe_script = tmp_file.name
tmp_file.write("#!/bin/sh\nexit 0")
os.chmod(check_exe_script, S_IRUSR | S_IWUSR | S_IEXEC)
subprocess.check_call([check_exe_script])
except (OSError, IOError, subprocess.CalledProcessError) as err:
LOGGER.debug(err)
continue
try:
statvfs = os.statvfs(check_exe_script)
except (OSError, IOError) as err:
LOGGER.debug(err)
if candidate is None:
candidate = tmp_prefix
continue
else:
free_mib = (statvfs.f_frsize*statvfs.f_bavail)/0x100000
LOGGER.debug("%s: %sMB free", tmp_prefix, free_mib)
if free_mib < 2000:
continue
if check_exe_script:
os.remove(check_exe_script)
break
else:
if not candidate:
raise ConfigurationError("No filesystem has at least 2GB free space and supports executables.")
tmp_prefix = candidate
LOGGER.warning("Unable to count available bytes in '%s'", tmp_prefix)
tmpfs_prefix.value = tmp_prefix
LOGGER.debug("Temporary prefix: '%s'", tmp_prefix)
return tmp_prefix