def main(self, argv):
TauInstallation.check_env_compat()
args = self._parse_args(argv)
store = arguments.parse_storage_flag(args)[0]
if hasattr(args, "forced_makefile"):
self._parse_tau_makefile(args)
self.logger.debug('Arguments after parsing TAU Makefile: %s', args)
compilers = self.parse_compiler_flags(args)
data = {attr: getattr(args, attr) for attr in self.model.attributes if hasattr(args, attr)}
for comp in compilers.itervalues():
record = Compiler.controller(store).register(comp)
data[comp.info.role.keyword] = record.eid
return super(TargetCreateCommand, self)._create_record(store, data)
def main(self, argv):
args = self._parse_args(argv)
profile_tools = getattr(args, 'profile_tools', None)
trace_tools = getattr(args, 'trace_tools', None)
data_files = []
trial_numbers = []
for num in getattr(args, 'trial_numbers', []) + getattr(
args, 'data_files', []):
if os.path.exists(num):
data_files.append(num)
else:
try:
trial_numbers.append(int(num))
except ValueError:
self.parser.error("Invalid trial number: %s" % num)
tau = TauInstallation.get_minimal()
dataset = {}
if not (data_files or trial_numbers):
expr = Project.selected().experiment()
for fmt, path in expr.trials()[0].get_data_files().iteritems():
dataset[fmt] = [path]
elif trial_numbers:
expr = Project.selected().experiment()
for trial in expr.trials(trial_numbers):
for fmt, path in trial.get_data_files().iteritems():
dataset.setdefault(fmt, []).append(path)
for path in data_files:
fmt = tau.get_data_format(path)
dataset.setdefault(fmt, []).append(path)
return tau.show_data_files(dataset, profile_tools, trace_tools)
def main(self, argv):
args = self._parse_args(argv)
profile_tools = getattr(args, 'profile_tools', None)
trace_tools = getattr(args, 'trace_tools', None)
data_files = []
trial_numbers = []
for num in getattr(args, 'trial_numbers', []) + getattr(args, 'data_files', []):
if os.path.exists(num):
data_files.append(num)
else:
try:
trial_numbers.append(int(num))
except ValueError:
self.parser.error("Invalid trial number: %s" % num)
tau = TauInstallation.get_minimal()
dataset = {}
if not (data_files or trial_numbers):
expr = Project.selected().experiment()
for fmt, path in expr.trials()[0].get_data_files().iteritems():
dataset[fmt] = [path]
elif trial_numbers:
expr = Project.selected().experiment()
for trial in expr.trials(trial_numbers):
for fmt, path in trial.get_data_files().iteritems():
dataset.setdefault(fmt, []).append(path)
for path in data_files:
fmt = tau.get_data_format(path)
dataset.setdefault(fmt, []).append(path)
return tau.show_data_files(dataset, profile_tools, trace_tools)
def main(self, argv):
TauInstallation.check_env_compat()
args = self._parse_args(argv)
store = arguments.parse_storage_flag(args)[0]
compilers = target_create_cmd.parse_compiler_flags(args)
data = {attr: getattr(args, attr) for attr in self.model.attributes if hasattr(args, attr)}
for keyword, comp in compilers.iteritems():
self.logger.debug("%s=%s (%s)", keyword, comp.absolute_path, comp.info.short_descr)
record = Compiler.controller(store).register(comp)
data[comp.info.role.keyword] = record.eid
key_attr = self.model.key_attribute
try:
data[key_attr] = getattr(args, 'copy_%s' % key_attr)
except AttributeError:
pass
key = getattr(args, key_attr)
return self._copy_record(store, data, key)
def main(self, argv):
TauInstallation.check_env_compat()
args = self._parse_args(argv)
store = arguments.parse_storage_flag(args)[0]
if hasattr(args, "forced_makefile"):
self._parse_tau_makefile(args)
self.logger.debug('Arguments after parsing TAU Makefile: %s', args)
compilers = self.parse_compiler_flags(args)
data = {
attr: getattr(args, attr)
for attr in self.model.attributes if hasattr(args, attr)
}
for comp in compilers.itervalues():
record = Compiler.controller(store).register(comp)
data[comp.info.role.keyword] = record.eid
return super(TargetCreateCommand, self)._create_record(store, data)
def export(self, dest):
"""Export experiment trial data.
Args:
dest (str): Path to directory to contain exported data.
Raises:
ConfigurationError: This trial has no data.
"""
expr = self.populate('experiment')
if self.get('data_size', 0) <= 0:
raise ConfigurationError(
"Trial %s of experiment '%s' has no data" %
(self['number'], expr['name']))
data = self.get_data_files()
stem = '%s.trial%d' % (expr['name'], self['number'])
for fmt, path in data.iteritems():
if fmt == 'tau':
export_file = os.path.join(dest, stem + '.ppk')
tau = TauInstallation.get_minimal()
tau.create_ppk_file(export_file, path)
elif fmt == 'merged':
export_file = os.path.join(dest, stem + '.xml.gz')
util.create_archive('gz', export_file, [path])
elif fmt == 'cubex':
export_file = os.path.join(dest, stem + '.cubex')
LOGGER.info("Writing '%s'...", export_file)
util.copy_file(path, export_file)
elif fmt == 'slog2':
export_file = os.path.join(dest, stem + '.slog2')
LOGGER.info("Writing '%s'...", export_file)
util.copy_file(path, export_file)
elif fmt == 'otf2':
export_file = os.path.join(dest, stem + '.tgz')
expr_dir, trial_dir = os.path.split(os.path.dirname(path))
items = [
os.path.join(trial_dir, item)
for item in 'traces', 'traces.def', 'traces.otf2'
]
util.create_archive('tgz', export_file, items, expr_dir)
elif fmt != 'none':
raise InternalError("Unhandled data file format '%s'" % fmt)
def _postprocess_slog2(self):
slog2 = os.path.join(self.prefix, 'tau.slog2')
if os.path.exists(slog2):
return
tau = TauInstallation.get_minimal()
merged_trc = os.path.join(self.prefix, 'tau.trc')
merged_edf = os.path.join(self.prefix, 'tau.edf')
if not os.path.exists(merged_trc) or not os.path.exists(merged_edf):
tau.merge_tau_trace_files(self.prefix)
tau.tau_trace_to_slog2(merged_trc, merged_edf, slog2)
trc_files = glob.glob(os.path.join(self.prefix, '*.trc'))
edf_files = glob.glob(os.path.join(self.prefix, '*.edf'))
count_trc_edf = len(trc_files) + len(edf_files)
LOGGER.info('Cleaning up TAU trace files...')
with ProgressIndicator("", total_size=count_trc_edf) as progress_bar:
count = 0
for path in trc_files + edf_files:
os.remove(path)
count += 1
progress_bar.update(count)
def _postprocess_slog2(self):
slog2 = os.path.join(self.prefix, 'tau.slog2')
if os.path.exists(slog2):
return
tau = TauInstallation.get_minimal()
merged_trc = os.path.join(self.prefix, 'tau.trc')
merged_edf = os.path.join(self.prefix, 'tau.edf')
if not os.path.exists(merged_trc) or not os.path.exists(merged_edf):
tau.merge_tau_trace_files(self.prefix)
tau.tau_trace_to_slog2(merged_trc, merged_edf, slog2)
trc_files = glob.glob(os.path.join(self.prefix, '*.trc'))
edf_files = glob.glob(os.path.join(self.prefix, '*.edf'))
count_trc_edf = len(trc_files) + len(edf_files)
LOGGER.info('Cleaning up TAU trace files...')
with ProgressIndicator(count_trc_edf) as progress_bar:
count = 0
for path in trc_files + edf_files:
os.remove(path)
count += 1
progress_bar.update(count)
def export(self, dest):
"""Export experiment trial data.
Args:
dest (str): Path to directory to contain exported data.
Raises:
ConfigurationError: This trial has no data.
"""
expr = self.populate('experiment')
if self.get('data_size', 0) <= 0:
raise ConfigurationError("Trial %s of experiment '%s' has no data" % (self['number'], expr['name']))
data = self.get_data_files()
stem = '%s.trial%d' % (expr['name'], self['number'])
for fmt, path in data.iteritems():
if fmt == 'tau':
export_file = os.path.join(dest, stem+'.ppk')
tau = TauInstallation.get_minimal()
tau.create_ppk_file(export_file, path)
elif fmt == 'merged':
export_file = os.path.join(dest, stem+'.xml.gz')
util.create_archive('gz', export_file, [path])
elif fmt == 'cubex':
export_file = os.path.join(dest, stem+'.cubex')
LOGGER.info("Writing '%s'...", export_file)
util.copy_file(path, export_file)
elif fmt == 'slog2':
export_file = os.path.join(dest, stem+'.slog2')
LOGGER.info("Writing '%s'...", export_file)
util.copy_file(path, export_file)
elif fmt == 'otf2':
export_file = os.path.join(dest, stem+'.tgz')
expr_dir, trial_dir = os.path.split(os.path.dirname(path))
items = [os.path.join(trial_dir, item) for item in 'traces', 'traces.def', 'traces.otf2']
util.create_archive('tgz', export_file, items, expr_dir)
elif fmt != 'none':
raise InternalError("Unhandled data file format '%s'" % fmt)
def configure(self):
"""Sets up the Experiment for a new trial.
Installs or configures TAU and all its dependencies. After calling this
function, the experiment is ready to operate on the user's application.
Returns:
TauInstallation: Object handle for the TAU installation.
"""
from taucmdr.cf.software.tau_installation import TauInstallation
LOGGER.debug("Configuring experiment %s", self['name'])
with fasteners.InterProcessLock(
os.path.join(PROJECT_STORAGE.prefix, '.lock')):
populated = self.populate(defaults=True)
target = populated['target']
application = populated['application']
measurement = populated['measurement']
baseline = measurement.get_or_default('baseline')
tau = TauInstallation(\
target.sources(),
target_arch=target.architecture(),
target_os=target.operating_system(),
compilers=target.compilers(),
# Use a minimal configuration for the baseline measurement
minimal=baseline,
# TAU feature suppport
application_linkage=application.get_or_default('linkage'),
openmp_support=application.get_or_default('openmp'),
pthreads_support=application.get_or_default('pthreads'),
tbb_support=application.get_or_default('tbb'),
mpi_support=application.get_or_default('mpi'),
mpi_libraries=target.get('mpi_libraries', []),
cuda_support=application.get_or_default('cuda'),
cuda_prefix=target.get('cuda_toolkit', None),
opencl_support=application.get_or_default('opencl'),
opencl_prefix=target.get('opencl', None),
shmem_support=application.get_or_default('shmem'),
shmem_libraries=target.get('shmem_libraries', []),
mpc_support=application.get_or_default('mpc'),
# Instrumentation methods and options
source_inst=measurement.get_or_default('source_inst'),
compiler_inst=measurement.get_or_default('compiler_inst'),
keep_inst_files=measurement.get_or_default('keep_inst_files'),
reuse_inst_files=measurement.get_or_default('reuse_inst_files'),
select_file=application.get('select_file', None),
# Measurement methods and options
baseline=baseline,
profile=measurement.get_or_default('profile'),
trace=measurement.get_or_default('trace'),
sample=measurement.get_or_default('sample'),
metrics=measurement.get_or_default('metrics'),
measure_io=measurement.get_or_default('io'),
measure_mpi=measurement.get_or_default('mpi'),
measure_openmp=measurement.get_or_default('openmp'),
measure_opencl=measurement.get_or_default('opencl'),
measure_cuda=measurement.get_or_default('cuda'),
measure_shmem=measurement.get_or_default('shmem'),
measure_heap_usage=measurement.get_or_default('heap_usage'),
measure_system_load=measurement.get_or_default('system_load'),
measure_memory_alloc=measurement.get_or_default('memory_alloc'),
measure_comm_matrix=measurement.get_or_default('comm_matrix'),
measure_callsite=measurement.get_or_default('callsite'),
callpath_depth=measurement.get_or_default('callpath'),
throttle=measurement.get_or_default('throttle'),
metadata_merge=measurement.get_or_default('metadata_merge'),
throttle_per_call=measurement.get_or_default('throttle_per_call'),
throttle_num_calls=measurement.get_or_default('throttle_num_calls'),
forced_makefile=target.get('forced_makefile', None))
tau.install()
if not baseline:
self.controller(self.storage).update(
{'tau_makefile': os.path.basename(tau.get_makefile())},
self.eid)
return tau
def managed_rewrite(self, rewrite_package, executable, inst_file):
from taucmdr.cf.software.tau_installation import TauInstallation
with fasteners.InterProcessLock(os.path.join(PROJECT_STORAGE.prefix, '.lock')):
populated = self.populate(defaults=True)
target = populated['target']
application = populated['application']
measurement = populated['measurement']
if rewrite_package == 'maqao' or rewrite_package == 'pebil':
source_inst = "automatic"
dyninst = False
else:
source_inst = 'never'
dyninst = True
tau = TauInstallation(\
target.sources(),
target_arch=target.architecture(),
target_os=target.operating_system(),
compilers=target.compilers(),
# TAU feature suppport
application_linkage=application.get_or_default('linkage'),
openmp_support=application.get_or_default('openmp'),
pthreads_support=application.get_or_default('pthreads'),
tbb_support=application.get_or_default('tbb'),
mpi_support=application.get_or_default('mpi'),
mpi_libraries=target.get('mpi_libraries', []),
caf_support=application.get_or_default('caf'),
cuda_support=application.get_or_default('cuda'),
cuda_prefix=target.get('cuda_toolkit', None),
opencl_support=application.get_or_default('opencl'),
opencl_prefix=target.get('opencl', None),
shmem_support=application.get_or_default('shmem'),
shmem_libraries=target.get('shmem_libraries', []),
mpc_support=application.get_or_default('mpc'),
max_threads=application.get('max_threads', None),
# Instrumentation methods and options
source_inst=source_inst,
compiler_inst=measurement.get_or_default('compiler_inst'),
keep_inst_files=measurement.get_or_default('keep_inst_files'),
reuse_inst_files=measurement.get_or_default('reuse_inst_files'),
select_file=measurement.get('select_file', None),
# Measurement methods and options
profile=measurement.get_or_default('profile'),
trace=measurement.get_or_default('trace'),
sample=measurement.get_or_default('sample'),
metrics=measurement.get_or_default('metrics'),
measure_io=measurement.get_or_default('io'),
measure_mpi=measurement.get_or_default('mpi'),
measure_openmp=measurement.get_or_default('openmp'),
measure_opencl=measurement.get_or_default('opencl'),
measure_cuda=measurement.get_or_default('cuda'),
measure_shmem=measurement.get_or_default('shmem'),
measure_heap_usage=measurement.get_or_default('heap_usage'),
measure_system_load=measurement.get_or_default('system_load'),
measure_memory_alloc=measurement.get_or_default('memory_alloc'),
measure_comm_matrix=measurement.get_or_default('comm_matrix'),
measure_callsite=measurement.get_or_default('callsite'),
callpath_depth=measurement.get_or_default('callpath'),
throttle=measurement.get_or_default('throttle'),
metadata_merge=measurement.get_or_default('metadata_merge'),
throttle_per_call=measurement.get_or_default('throttle_per_call'),
throttle_num_calls=measurement.get_or_default('throttle_num_calls'),
sampling_period=measurement.get_or_default('sampling_period'),
track_memory_footprint=measurement.get_or_default('track_memory_footprint'),
update_nightly=measurement.get_or_default('update_nightly'),
ptts=measurement.get_or_default('ptts'),
ptts_post=measurement.get_or_default('ptts_post'),
ptts_sample_flags=measurement.get_or_default('ptts_sample_flags'),
ptts_restart=measurement.get_or_default('ptts_restart'),
ptts_start=measurement.get_or_default('ptts_start'),
ptts_stop=measurement.get_or_default('ptts_stop'),
ptts_report_flags=measurement.get_or_default('ptts_report_flags'),
forced_makefile=target.get('forced_makefile', None),
mpit=measurement.get_or_default('mpit'),
unwind_depth=measurement.get_or_default('unwind_depth'),
dyninst=dyninst)
tau.install()
tau.rewrite(rewrite_package, executable, inst_file)
return tau
def configure(self):
"""Sets up the Experiment for a new trial.
Installs or configures TAU and all its dependencies. After calling this
function, the experiment is ready to operate on the user's application.
Returns:
TauInstallation: Object handle for the TAU installation.
"""
from taucmdr.cf.software.tau_installation import TauInstallation
LOGGER.debug("Configuring experiment %s", self['name'])
with fasteners.InterProcessLock(os.path.join(PROJECT_STORAGE.prefix, '.lock')):
populated = self.populate(defaults=True)
target = populated['target']
application = populated['application']
measurement = populated['measurement']
baseline = measurement.get_or_default('baseline')
tau = TauInstallation(\
target.sources(),
target_arch=target.architecture(),
target_os=target.operating_system(),
compilers=target.compilers(),
# Use a minimal configuration for the baseline measurement
minimal=baseline,
# TAU feature suppport
application_linkage=application.get_or_default('linkage'),
openmp_support=application.get_or_default('openmp'),
pthreads_support=application.get_or_default('pthreads'),
tbb_support=application.get_or_default('tbb'),
mpi_support=application.get_or_default('mpi'),
mpi_libraries=target.get('mpi_libraries', []),
cuda_support=application.get_or_default('cuda'),
cuda_prefix=target.get('cuda_toolkit', None),
opencl_support=application.get_or_default('opencl'),
opencl_prefix=target.get('opencl', None),
shmem_support=application.get_or_default('shmem'),
shmem_libraries=target.get('shmem_libraries', []),
mpc_support=application.get_or_default('mpc'),
# Instrumentation methods and options
source_inst=measurement.get_or_default('source_inst'),
compiler_inst=measurement.get_or_default('compiler_inst'),
keep_inst_files=measurement.get_or_default('keep_inst_files'),
reuse_inst_files=measurement.get_or_default('reuse_inst_files'),
select_file=application.get('select_file', None),
# Measurement methods and options
baseline=baseline,
profile=measurement.get_or_default('profile'),
trace=measurement.get_or_default('trace'),
sample=measurement.get_or_default('sample'),
metrics=measurement.get_or_default('metrics'),
measure_io=measurement.get_or_default('io'),
measure_mpi=measurement.get_or_default('mpi'),
measure_openmp=measurement.get_or_default('openmp'),
measure_opencl=measurement.get_or_default('opencl'),
measure_cuda=measurement.get_or_default('cuda'),
measure_shmem=measurement.get_or_default('shmem'),
measure_heap_usage=measurement.get_or_default('heap_usage'),
measure_system_load=measurement.get_or_default('system_load'),
measure_memory_alloc=measurement.get_or_default('memory_alloc'),
measure_comm_matrix=measurement.get_or_default('comm_matrix'),
measure_callsite=measurement.get_or_default('callsite'),
callpath_depth=measurement.get_or_default('callpath'),
throttle=measurement.get_or_default('throttle'),
metadata_merge=measurement.get_or_default('metadata_merge'),
throttle_per_call=measurement.get_or_default('throttle_per_call'),
throttle_num_calls=measurement.get_or_default('throttle_num_calls'),
forced_makefile=target.get('forced_makefile', None))
tau.install()
if not baseline:
self.controller(self.storage).update({'tau_makefile': os.path.basename(tau.get_makefile())}, self.eid)
return tau
