def main(self, argv):
args = self._parse_args(argv)
profile_tools = getattr(args, 'profile_tools', None)
trace_tools = getattr(args, 'trace_tools', None)
data_files = []
trial_numbers = []
for num in getattr(args, 'trial_numbers', []) + getattr(args, 'data_files', []):
if os.path.exists(num):
data_files.append(num)
else:
try:
trial_numbers.append(int(num))
except ValueError:
self.parser.error("Invalid trial number: %s" % num)
tau = TauInstallation.get_minimal()
dataset = {}
if not (data_files or trial_numbers):
expr = Project.selected().experiment()
for fmt, path in expr.trials()[0].get_data_files().iteritems():
dataset[fmt] = [path]
elif trial_numbers:
expr = Project.selected().experiment()
for trial in expr.trials(trial_numbers):
for fmt, path in trial.get_data_files().iteritems():
dataset.setdefault(fmt, []).append(path)
for path in data_files:
fmt = tau.get_data_format(path)
dataset.setdefault(fmt, []).append(path)
return tau.show_data_files(dataset, profile_tools, trace_tools)
def main(self, argv):
args = self._parse_args(argv)
if args.tau_dir:
# Link to tau directory
if argv[0] in '--tau-dir':
del argv[0:2]
elif argv[2] in '--tau-dir':
del argv[2:4]
if args.description:
description = args.description
if argv[0] in '--description':
del argv[0:2]
elif argv[2] in '--description':
del argv[2:4]
else:
description = None
launcher_cmd, application_cmds = Trial.parse_launcher_cmd(argv)
self.logger.debug("Launcher command: %s", launcher_cmd)
self.logger.debug("Application commands: %s", application_cmds)
if args.tau_dir:
Project.controller().storage.tau_dir(args.tau_dir)
return Project.selected().experiment().managed_run(
launcher_cmd, application_cmds, description)
return Project.selected().experiment().managed_run(
launcher_cmd, application_cmds, description)
def get_pandas_non_summary():
'''
returns a dictionary of pandas
keys are the metrics that each panda has data for
vals are the pandas that have the data organized however they organzed it
DEPRECATED - may not work
'''
num_trials = Project.selected().experiment().num_trials
trials = Project.selected().experiment().trials(xrange(0, num_trials))
trial_data = {}
for i in xrange(0, num_trials):
trial_data[i] = trials[i].get_data()
start = time.time()
metric_data = {}
for trial in xrange(0, num_trials):
thread_data = []
for i in xrange(0, len(trial_data[trial])):
for j in xrange(0, len(trial_data[trial][i])):
for k in xrange(0, len(trial_data[trial][i][j])):
thread_data.append(trial_data[trial][i][j][k].interval_data())
metric_data[trial_data[trial][i][j][k].metric] = pd.concat(thread_data)
metric_data[trial_data[trial][i][j][k].metric].index.names = ['trial', 'rank', 'context', 'thread', 'region']
end = time.time()
print('Time spent constructing dataframes %s' %(end-start))
print('\nMetrics included:')
for m in metric_data.keys():
print("\t%s"%m)
return metric_data
def main(self, argv):
args = self._parse_args(argv)
profile_tools = getattr(args, 'profile_tools', None)
trace_tools = getattr(args, 'trace_tools', None)
data_files = []
trial_numbers = []
for num in getattr(args, 'trial_numbers', []) + getattr(
args, 'data_files', []):
if os.path.exists(num):
data_files.append(num)
else:
try:
trial_numbers.append(int(num))
except ValueError:
self.parser.error("Invalid trial number: %s" % num)
tau = TauInstallation.get_minimal()
dataset = {}
if not (data_files or trial_numbers):
expr = Project.selected().experiment()
for fmt, path in expr.trials()[0].get_data_files().iteritems():
dataset[fmt] = [path]
elif trial_numbers:
expr = Project.selected().experiment()
for trial in expr.trials(trial_numbers):
for fmt, path in trial.get_data_files().iteritems():
dataset.setdefault(fmt, []).append(path)
for path in data_files:
fmt = tau.get_data_format(path)
dataset.setdefault(fmt, []).append(path)
return tau.show_data_files(dataset, profile_tools, trace_tools)
def controller(cls, storage=PROJECT_STORAGE):
if Project.selected():
context = [('project', Project.selected().eid),
('projects', Project.selected().eid)]
else:
# use a value that will never exist to block all
context = [('project', 'Undefined')]
return cls.__controller__(cls, storage, context)
def main(self, argv):
"""Program entry point.
Args:
argv (list): Command line arguments.
Returns:
int: Process return code: non-zero if a problem occurred, 0 otherwise
"""
args = self._parse_args(argv)
cmd = args.command
cmd_args = args.options
log_level = getattr(args, 'verbose',
getattr(args, 'quiet', logger.LOG_LEVEL))
logger.set_log_level(log_level)
LOGGER.debug('Arguments: %s', args)
LOGGER.debug('Verbosity level: %s', logger.LOG_LEVEL)
# Try to execute as a TAU command
try:
return cli.execute_command([cmd], cmd_args)
except UnknownCommandError:
pass
# Check shortcuts
shortcut = None
from taucmdr.model.project import Project
uses_python = Project.selected().experiment().populate(
)['application'].get_or_default('python')
if not uses_python and build_command.is_compatible(
cmd): # should return false for python
shortcut = ['build']
cmd_args = [cmd] + cmd_args
elif trial_create_command.is_compatible(
cmd): # should return true for python
shortcut = ['trial', 'create']
cmd_args = [cmd] + cmd_args
elif 'show'.startswith(cmd):
shortcut = ['trial', 'show']
elif 'metrics'.startswith(cmd):
expr = Project.selected().experiment()
targ_name = expr.populate('target')['name']
shortcut = ['target', 'metrics']
cmd_args.insert(0, targ_name)
if shortcut:
LOGGER.debug('Trying shortcut: %s', shortcut)
return cli.execute_command(shortcut, cmd_args)
else:
LOGGER.debug('No shortcut found for %r', cmd)
# Not sure what to do at this point, so advise the user and exit
LOGGER.info("Unknown command. Calling `%s help %s` to get advice.",
TAUCMDR_SCRIPT, cmd)
return cli.execute_command(['help'], [cmd])
def _update_record(self, store, data, key):
proj_ctrl = Project.controller()
expr = Project.selected().experiment()
ctrl = self.model.controller(store)
key_attr = self.model.key_attribute
if not ctrl.exists({key_attr: key, 'experiment': expr.eid}):
self.parser.error(
"No %s-level %s with %s='%s'." %
(ctrl.storage.name, self.model_name, key_attr, key))
ctrl.update(data, {key_attr: key, 'experiment': expr.eid})
self.logger.info("Updated %s '%s'", self.model_name, key)
return EXIT_SUCCESS
def main(self, argv):
args = self._parse_args(argv)
if not (args.profile or args.trace or args.sample):
self.parser.error('You must specify at least one measurement.')
proj_ctrl = Project.controller()
try:
proj = proj_ctrl.selected()
except ProjectStorageError:
self.logger.debug("No project found, initializing a new project.")
PROJECT_STORAGE.connect_filesystem()
try:
self._create_project(args)
if not args.bare:
self._populate_project(argv, args)
except:
PROJECT_STORAGE.destroy()
raise
return dashboard_cmd.main([])
except ProjectSelectionError as err:
err.value = "The project has been initialized but no project configuration is selected."
raise err
else:
self.logger.warning("Tau is already initialized and the selected project is '%s'. Use commands like"
" `tau application edit` to edit the selected project or delete"
" '%s' to reset to a fresh environment.", proj['name'], proj_ctrl.storage.prefix)
return EXIT_WARNING
def main(self, argv):
args = self._parse_args(argv)
store = arguments.parse_storage_flag(args)[0]
data = {
attr: getattr(args, attr)
for attr in self.model.attributes if hasattr(args, attr)
}
# Translate names to eids
proj_ctrl = Project.controller()
for model in Target, Application, Measurement:
attr = model.name.lower()
if attr in data:
record = model.controller(proj_ctrl.storage).one(
{'name': data[attr]})
if record is None:
self.parser.error("Invalid %s: %s" % (attr, data[attr]))
data[attr] = record.eid
key_attr = self.model.key_attribute
try:
data[key_attr] = args.new_key
except AttributeError:
pass
key = getattr(args, key_attr)
return self._update_record(store, data, key)
def main(self, argv):
"""Command program entry point.
Args:
argv (list): Command line arguments.
Returns:
int: Process return code: non-zero if a problem occurred, 0 otherwise
"""
args = self._parse_args(argv)
if args.current:
proj = Project.controller().selected()
try:
expr = proj.experiment()
except ExperimentSelectionError:
print(
util.color_text('No selected experiment: ', 'red') +
'Use `%s` to create or select an experiment.' % select_cmd)
else:
print expr['name']
retval = EXIT_SUCCESS
else:
retval = super(ExperimentListCommand, self).main(argv)
return retval
def test_heap_usage_memory_alloc_profile(self):
"""https://github.com/ParaToolsInc/taucmdr/issues/14"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(
0, measurement_edit_cmd,
['profile', '--heap-usage', '--memory-alloc'])
self.assertIn("Updated measurement 'profile'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd,
['profile'])
self.assertIn("Selected experiment 'targ1-app1-profile'", stdout)
self.assertFalse(stderr)
meas = Project.selected().experiment().populate('measurement')
self.assertTrue(meas['heap_usage'])
self.assertTrue(meas['memory_alloc'])
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd,
['./a.out'])
self.assertIn("Trial 0 produced 1 profile files", stdout)
self.assertIn("TAU_SHOW_MEMORY_FUNCTIONS=1", stdout)
self.assertIn("TAU_TRACK_HEAP=1", stdout)
self.assertFalse(stderr)
self.assertInLastTrialData(
"<attribute><name>TAU_SHOW_MEMORY_FUNCTIONS</name><value>on</value></attribute>"
)
self.assertInLastTrialData(
"<attribute><name>TAU_TRACK_HEAP</name><value>on</value></attribute>"
)
self.assertInLastTrialData("Heap Memory Used (KB) at Entry")
self.assertInLastTrialData("Heap Memory Used (KB) at Exit")
self.assertInLastTrialData("Heap Allocate")
self.assertInLastTrialData("compute_interchange")
self.assertInLastTrialData("compute")
self.assertInLastTrialData("malloc")
def _create_record(self, store, data):
"""Create the model record.
Args:
store (AbstractStorage): Storage to contain the record.
data (dict): Record data.
Returns:
int: :any:`EXIT_SUCCESS` if successful.
Raises:
UniqueAttributeError: A record with the same unique attribute already exists.
"""
ctrl = self.model.controller(store)
key_attr = self.model.key_attribute
key = data[key_attr]
try:
ctrl.create(data)
except UniqueAttributeError:
self.parser.error("A %s with %s='%s' already exists" %
(self.model_name, key_attr, key))
if ctrl.storage is PROJECT_STORAGE:
from taucmdr.cli.commands.project.edit import COMMAND as project_edit_cmd
try:
proj = Project.selected()
except ProjectSelectionError:
self.logger.info(
"Created a new %s '%s'. Use `%s` to add the new %s to a project.",
self.model_name, key, project_edit_cmd, self.model_name)
else:
project_edit_cmd.main([proj['name'], '--add', key])
else:
self.logger.info("Created a new %s-level %s: '%s'.",
ctrl.storage.name, self.model_name, key)
return EXIT_SUCCESS
def main(self, argv):
if not argv:
self.parser.error("too few arguments.")
args = self._parse_args(argv)
proj = Project.selected()
targ, app, meas, expr = self._parse_implicit(args)
expr = self._parse_explicit_experiment(args, expr)
if expr:
targ, app, meas = None, None, None
name = expr['name']
else:
targ = self._parse_explicit(args, Target, targ, proj, 'targets')
app = self._parse_explicit(args, Application, app, proj, 'applications')
meas = self._parse_explicit(args, Measurement, meas, proj, 'measurements')
name = getattr(args, 'name', "%s-%s-%s" % (targ['name'], app['name'], meas['name']))
try:
Experiment.select(name)
except ExperimentSelectionError:
args = [name,
'--target', targ['name'],
'--application', app['name'],
'--measurement', meas['name']]
retval = experiment_create_cmd.main(args)
if retval != EXIT_SUCCESS:
return retval
Experiment.select(name)
self.logger.info("Selected experiment '%s'.", name)
rebuild_required = Experiment.rebuild_required()
if rebuild_required:
self.logger.info(rebuild_required)
return EXIT_SUCCESS
def _create_record(self, store, data):
"""Create the model record.
Args:
store (AbstractStorage): Storage to contain the record.
data (dict): Record data.
Returns:
int: :any:`EXIT_SUCCESS` if successful.
Raises:
UniqueAttributeError: A record with the same unique attribute already exists.
"""
ctrl = self.model.controller(store)
key_attr = self.model.key_attribute
key = data[key_attr]
try:
ctrl.create(data)
except UniqueAttributeError:
self.parser.error("A %s with %s='%s' already exists" % (self.model_name, key_attr, key))
if ctrl.storage is PROJECT_STORAGE:
from taucmdr.cli.commands.project.edit import COMMAND as project_edit_cmd
try:
proj = Project.selected()
except ProjectSelectionError:
self.logger.info("Created a new %s '%s'. Use `%s` to add the new %s to a project.",
self.model_name, key, project_edit_cmd, self.model_name)
else:
project_edit_cmd.main([proj['name'], '--add', key])
else:
self.logger.info("Created a new %s-level %s: '%s'.", ctrl.storage.name, self.model_name, key)
return EXIT_SUCCESS
def renumber(self, old_trials, new_trials):
"""Renumbers trial id of an experiment.
Args:
old_trials (list): old trial numbers.
new_trials (list): new trial numbers.
"""
# First, we renumber everything to available temporary numbers
assert len(old_trials) == len(new_trials)
expr = Project.selected().experiment()
existing_nums = [
trial['number'] for trial in Trial.controller(
storage=PROJECT_STORAGE).search({'experiment': expr.eid})
]
start_temp_id = max(max(existing_nums), max(new_trials)) + 1
temp_id = start_temp_id
for old_trial_num in old_trials:
old_trial = self.one({
'number': old_trial_num,
'experiment': expr.eid
})
self.update({'number': temp_id}, old_trial.eid)
temp_id = temp_id + 1
# Then we renumber from the temporaries to the final new numbers
temp_id = start_temp_id
for new_trial_num in new_trials:
intermed_trial = self.one({
'number': temp_id,
'experiment': expr.eid
})
self.update({'number': new_trial_num}, intermed_trial.eid)
temp_id = temp_id + 1
def main(self, argv):
self._parse_args(argv)
launcher_cmd, application_cmds = Trial.parse_launcher_cmd(argv)
self.logger.debug("Launcher command: %s", launcher_cmd)
self.logger.debug("Application commands: %s", application_cmds)
return Project.selected().experiment().managed_run(
launcher_cmd, application_cmds)
def main(self, argv):
args = self._parse_args(argv)
if not (args.baseline or args.profile or args.trace or args.sample):
self.parser.error('You must specify at least one measurement.')
proj_ctrl = Project.controller()
try:
proj = proj_ctrl.selected()
except ProjectStorageError:
self.logger.debug("No project found, initializing a new project.")
PROJECT_STORAGE.connect_filesystem()
try:
self._create_project(args)
if not args.bare:
self._populate_project(args)
except:
PROJECT_STORAGE.destroy()
raise
return dashboard_cmd.main([])
except ProjectSelectionError as err:
err.value = "The project has been initialized but no project configuration is selected."
raise err
else:
self.logger.warning(
"Tau is already initialized and the selected project is '%s'. Use commands like"
" `tau application edit` to edit the selected project or delete"
" '%s' to reset to a fresh environment.", proj['name'],
proj_ctrl.storage.prefix)
return EXIT_WARNING
def main(self, argv):
if not argv:
self.parser.error("too few arguments.")
args = self._parse_args(argv)
proj = Project.selected()
targ, app, meas, expr = self._parse_implicit(args)
expr = self._parse_explicit_experiment(args, expr)
if expr:
targ, app, meas = None, None, None
name = expr['name']
else:
targ = self._parse_explicit(args, Target, targ, proj, 'targets')
app = self._parse_explicit(args, Application, app, proj,
'applications')
meas = self._parse_explicit(args, Measurement, meas, proj,
'measurements')
name = getattr(
args, 'name',
"%s-%s-%s" % (targ['name'], app['name'], meas['name']))
try:
Experiment.select(name)
except ExperimentSelectionError:
args = [
name, '--target', targ['name'], '--application', app['name'],
'--measurement', meas['name']
]
retval = experiment_create_cmd.main(args)
if retval != EXIT_SUCCESS:
return retval
Experiment.select(name)
self.logger.info("Selected experiment '%s'.", name)
rebuild_required = Experiment.rebuild_required()
if rebuild_required:
self.logger.info(rebuild_required)
return EXIT_SUCCESS
def main(self, argv):
args = self._parse_args(argv)
proj_ctrl = Project.controller()
proj = proj_ctrl.selected()
targ = Target.controller(proj_ctrl.storage).one({'name': args.target})
if targ is None:
self.parser.error("A target with name %s does not exist." %
args.target)
app = Application.controller(proj_ctrl.storage).one(
{'name': args.application})
if app is None:
self.parser.error("An application with name %s does not exist." %
args.application)
meas = Measurement.controller(proj_ctrl.storage).one(
{'name': args.measurement})
if meas is None:
self.parser.error("A measurement with name %s does not exist." %
args.measurement)
data = {
'name': args.name,
'project': proj.eid,
'target': targ.eid,
'application': app.eid,
'measurement': meas.eid
}
return self._create_record(PROJECT_STORAGE, data)
def test_heap_usage_memory_alloc_profile(self):
"""https://github.com/ParaToolsInc/taucmdr/issues/14"""
self.reset_project_storage()
stdout, stderr = self.assertCommandReturnValue(0, measurement_edit_cmd,
['profile', '--heap-usage', '--memory-alloc'])
self.assertIn("Updated measurement 'profile'", stdout)
self.assertFalse(stderr)
stdout, stderr = self.assertCommandReturnValue(0, select_cmd, ['profile'])
self.assertIn("Selected experiment 'targ1-app1-profile'", stdout)
self.assertFalse(stderr)
meas = Project.selected().experiment().populate('measurement')
self.assertTrue(meas['heap_usage'])
self.assertTrue(meas['memory_alloc'])
self.assertManagedBuild(0, CC, [], 'matmult.c')
stdout, stderr = self.assertCommandReturnValue(0, trial_create_cmd, ['./a.out'])
self.assertIn("Trial 0 produced 1 profile files", stdout)
self.assertIn("TAU_SHOW_MEMORY_FUNCTIONS=1", stdout)
self.assertIn("TAU_TRACK_HEAP=1", stdout)
self.assertFalse(stderr)
self.assertInLastTrialData("<attribute><name>TAU_SHOW_MEMORY_FUNCTIONS</name><value>on</value></attribute>")
self.assertInLastTrialData("<attribute><name>TAU_TRACK_HEAP</name><value>on</value></attribute>")
self.assertInLastTrialData("Heap Memory Used (KB) at Entry")
self.assertInLastTrialData("Heap Memory Used (KB) at Exit")
self.assertInLastTrialData("Heap Allocate")
self.assertInLastTrialData("compute_interchange")
self.assertInLastTrialData("compute")
self.assertInLastTrialData("malloc")
def is_selected(self):
"""Returns True if this target configuration is part of the selected experiment, False otherwise."""
from taucmdr.model.project import Project
try:
selected = Project.selected().experiment()
except (ProjectSelectionError, ExperimentSelectionError):
return False
return selected['application'] == self.eid
def _project_eid(self):
"""Avoid multiple lookup of the selected project since project selection will not change mid-process."""
try:
return self._selected_project.eid
except AttributeError:
# pylint: disable=attribute-defined-outside-init
self._selected_project = Project.controller(self.storage).selected()
return self._selected_project.eid
def is_selected(self):
"""Returns True if this target configuration is part of the selected experiment, False otherwise."""
from taucmdr.model.project import Project
try:
selected = Project.selected().experiment()
except (ProjectSelectionError, ExperimentSelectionError):
return False
return selected['application'] == self.eid
def test_rename(self):
self.reset_project_storage()
argv = ['proj1', '--new-name', 'proj2']
self.assertCommandReturnValue(0, edit.COMMAND, argv)
proj_ctrl = Project.controller()
self.assertIsNone(proj_ctrl.one({'name': 'proj1'}))
self.assertIsNotNone(proj_ctrl.one({'name': 'proj2'}))
self.assertCommandReturnValue(0, edit.COMMAND, ['proj2', '--new-name', 'proj1'])
def _project_eid(self):
"""Avoid multiple lookup of the selected project since project selection will not change mid-process."""
try:
return self._selected_project.eid
except AttributeError:
# pylint: disable=attribute-defined-outside-init
self._selected_project = Project.controller(self.storage).selected()
return self._selected_project.eid
def test_rename(self):
self.reset_project_storage()
argv = ['proj1', '--new-name', 'proj2']
self.assertCommandReturnValue(0, edit.COMMAND, argv)
proj_ctrl = Project.controller()
self.assertIsNone(proj_ctrl.one({'name': 'proj1'}))
self.assertIsNotNone(proj_ctrl.one({'name': 'proj2'}))
self.assertCommandReturnValue(0, edit.COMMAND, ['proj2', '--new-name', 'proj1'])
def main(self, argv):
args = self._parse_args(argv)
description = getattr(args, 'description', None)
cmd = [args.cmd] + args.cmd_args
launcher_cmd, application_cmds = Trial.parse_launcher_cmd(cmd)
self.logger.debug("Launcher command: %s", launcher_cmd)
self.logger.debug("Application commands: %s", application_cmds)
return Project.selected().experiment().managed_run(launcher_cmd, application_cmds, description)
def main(self, argv):
args = self._parse_args(argv)
proj_ctrl = Project.controller()
name = args.name
proj = proj_ctrl.one({"name": name})
if not proj:
self.parser.error("There is no project configuration named '%s.'" % name)
proj_ctrl.select(proj)
return EXIT_SUCCESS
def _retrieve_records(self, ctrl, keys):
if keys:
try:
keys = [int(key) for key in keys]
except ValueError:
self.parser.error("Invalid trial number '%s'. Trial numbers are positive integers starting from 0.")
expr = Project.selected().experiment()
records = super(TrialListCommand, self)._retrieve_records(ctrl, keys)
return [rec for rec in records if rec['experiment'] == expr.eid]
def test_newtrial(self):
self.reset_project_storage()
self.assertManagedBuild(0, CC, [], 'hello.c')
self.assertCommandReturnValue(0, CREATE_COMMAND, ['./a.out'])
self.assertCommandReturnValue(0, EDIT_COMMAND,
['0', '--description', 'desc0'])
exp = Project.selected().experiment().eid
old_path = Trial.controller(storage=PROJECT_STORAGE).search({
'number':
0,
'experiment':
exp
})[0].get_data_files()['tau']
self.assertTrue(os.path.exists(old_path),
"Data directory should exist after create")
old_profile = os.path.join(old_path, "profile.0.0.0")
self.assertTrue(os.path.exists(old_profile),
"Profile should exist after create")
with open(old_profile, 'r') as f:
old_profile_contents = f.read()
num_trials_before = Trial.controller(storage=PROJECT_STORAGE).count()
self.assertCommandReturnValue(0, RENUMBER_COMMAND, ['0', '--to', '1'])
stdout, stderr = self.assertCommandReturnValue(0, LIST_COMMAND, [])
self.assertIn('./a.out', stdout)
self.assertIn(' 1 ', stdout)
self.assertIn('desc0', stdout)
self.assertNotIn(' 0 ', stdout)
self.assertIn('Selected experiment:', stdout)
self.assertFalse(stderr)
num_trials_after = Trial.controller(storage=PROJECT_STORAGE).count()
self.assertEqual(num_trials_before, num_trials_after,
"Renumbering should not change number of trials")
self.assertFalse(
os.path.exists(old_path),
"Data directory for old number should not exist after renumber")
self.assertFalse(
os.path.exists(os.path.join(old_path, "profile.0.0.0")),
"Profile in old data directory should not exist after renumber")
new_path = Trial.controller(storage=PROJECT_STORAGE).search({
'number':
1,
'experiment':
exp
})[0].get_data_files()['tau']
self.assertTrue(
os.path.exists(new_path),
"Data directory for new number should exist after renumber")
new_profile = os.path.join(new_path, "profile.0.0.0")
self.assertTrue(
os.path.exists(new_profile),
"Profile in data directory for new number should exist after renumber"
)
with open(new_profile, 'r') as f:
new_profile_contents = f.read()
self.assertEqual(old_profile_contents, new_profile_contents,
"Profiles should be identical after renumber")
def test_set_tau_force_options(self):
self.reset_project_storage()
expr = Project.selected().experiment()
self.assertFalse('force-tau-options' in expr.populate('measurement'))
tau_options = "-optVerbose -optNoCompInst"
self.assertCommandReturnValue(0, EDIT_COMMAND, ['profile', '--force-tau-options=%s' % tau_options])
# Check that 'force-tau-options' is now a list containing the expected options in the project record
meas = Measurement.controller(PROJECT_STORAGE).one({'name': 'profile'})
self.assertIsNotNone(meas)
self.assertListEqual(meas['force_tau_options'], [tau_options])
def main(self, argv):
args = self._parse_args(argv)
proj_ctrl = Project.controller()
name = args.name
proj = proj_ctrl.one({"name": name})
if not proj:
self.parser.error("There is no project configuration named '%s.'" %
name)
proj_ctrl.select(proj)
return EXIT_SUCCESS
def _retrieve_records(self, ctrl, keys):
if keys:
try:
keys = [int(key) for key in keys]
except ValueError:
self.parser.error("Invalid trial number '%s'. Trial numbers are positive integers starting from 0.")
expr = Project.selected().experiment()
records = super(TrialListCommand, self)._retrieve_records(ctrl, keys)
recs = [rec for rec in records if rec['experiment'] == expr.eid]
return sorted(recs, key=lambda recs: recs['number'])
def test_set_tau_force_options(self):
self.reset_project_storage()
expr = Project.selected().experiment()
self.assertFalse('force-tau-options' in expr.populate('measurement'))
tau_options = "-optVerbose -optNoCompInst"
self.assertCommandReturnValue(0, EDIT_COMMAND, ['profile', '--force-tau-options=%s' % tau_options])
# Check that 'force-tau-options' is now a list containing the expected options in the project record
meas = Measurement.controller(PROJECT_STORAGE).one({'name': 'profile'})
self.assertIsNotNone(meas)
self.assertListEqual(meas['force_tau_options'], [tau_options])
def test_set_tau_extra_options_none(self):
self.reset_project_storage()
expr = Project.selected().experiment()
self.assertFalse('extra-tau-options' in expr.populate('measurement'))
tau_options = "none"
self.assertCommandReturnValue(0, EDIT_COMMAND, ['profile', '--extra-tau-options=%s' % tau_options])
# Check that 'extra-tau-options' is now a list containing the expected options in the project record
meas = Measurement.controller(PROJECT_STORAGE).one({'name': 'profile'})
self.assertIsNotNone(meas)
self.assertNotIn('extra_tau_options', meas)
def main(self, argv):
"""Command program entry point.
Args:
argv (list): Command line arguments.
Returns:
int: Process return code: non-zero if a problem occurred, 0 otherwise
"""
args = self._parse_args(argv)
style_args = ['--' + args.style] if hasattr(args, 'style') else []
levels = arguments.parse_storage_flag(args)
keys = getattr(args, 'keys', [])
single = (len(keys) == 1 and len(levels) == 1)
if single:
proj_name = keys[0]
self.title_fmt = "Project Configuration (%(storage_path)s)"
target_list_cmd.title_fmt = "Targets in project '%s'" % proj_name
application_list_cmd.title_fmt = "Applications in project '%s'" % proj_name
measurement_list_cmd.title_fmt = "Measurements in project '%s'" % proj_name
experiment_list_cmd.title_fmt = "Experiments in project '%s'" % proj_name
retval = super(ProjectListCommand, self).main(argv)
if single:
storage = levels[0]
ctrl = Project.controller(storage)
proj = ctrl.one({'name': keys[0]}, context=False)
for cmd, prop in ((target_list_cmd, 'targets'),
(application_list_cmd, 'applications'),
(measurement_list_cmd, 'measurements'),
(experiment_list_cmd, 'experiments')):
primary_key = proj.attributes[prop]['collection'].key_attribute
records = proj.populate(prop, context=False)
if records:
cmd.main([record[primary_key] for record in records] +
['-p'] + [proj['name']] + style_args)
else:
label = util.color_text('%s: No %s' % (proj['name'], prop),
color='red',
attrs=['bold'])
print "%s. Use `%s` to view available %s.\n" % (label,
cmd, prop)
try:
expr = proj.experiment()
if not isinstance(expr, NoneType):
print util.color_text("Selected Experiment: ",
'cyan') + expr['name']
except ExperimentSelectionError:
print(
util.color_text('No selected experiment: ', 'red') +
'Use `%s` to create or select an experiment.' % select_cmd)
return retval
def parse_launcher_cmd(cls, cmd):
"""Parses a command line to split the launcher command and application commands.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, possibly empty list of application commands).
"""
cmd0 = cmd[0]
launcher_cmd, cmd = cls._separate_launcher_cmd(cmd)
num_exes = len([x for x in cmd if util.which(x)])
assert launcher_cmd or cmd
LOGGER.debug('Launcher: %s', launcher_cmd)
LOGGER.debug('Remainder: %s', cmd)
uses_python = Project.selected().experiment().populate(
)['application'].get_or_default('python')
if uses_python:
if 'python' in cmd[0]:
cmd.remove(cmd[0])
if not launcher_cmd:
if num_exes > 1:
LOGGER.warning(
"Multiple executables were found on the command line but none of them "
"were recognized application launchers. TAU will assume that the application "
"executable is '%s' and subsequent executables are arguments to that command. "
"If this is incorrect, use '--' to separate '%s' and its arguments "
"from the application command, e.g. `mpirun -np 4 -- ./a.out -l hello`",
cmd0, cmd0)
return [], [cmd]
if not cmd:
return launcher_cmd, []
if num_exes <= 1:
return launcher_cmd, [cmd]
elif num_exes > 1:
colons = [i for i, x in enumerate(cmd) if x == ':']
if not colons:
# Recognized launcher with multiple executables but not using ':' syntax.
LOGGER.warning(
"Multiple executables were found on the command line. TAU will assume that "
"the application executable is '%s' and subsequent executables are arguments "
"to that command. If this is incorrect, use ':' to separate each application "
"executable and its arguments, e.g. `mpirun -np 4 ./foo -l : -np 2 ./bar arg1`. "
"Or, use '--' to separate '%s', its arguments, and subsequent executables "
"from the application command, e.g. "
"`mpirun -np 4 numactl -m 1 -- ./a.out -l hello", cmd0,
cmd0)
return launcher_cmd, [cmd]
# Split MPMD command on ':'. Retain ':' as first element of each application command
colons.append(len(cmd))
application_cmds = [cmd[:colons[0]]]
for i, idx in enumerate(colons[:-1]):
application_cmds.append(cmd[idx:colons[i + 1]])
return launcher_cmd, application_cmds
def test_set_tau_forced_extra_options(self):
self.reset_project_storage()
expr = Project.selected().experiment()
self.assertFalse('extra-tau-options' in expr.populate('measurement'))
self.assertFalse('forced-tau-options' in expr.populate('measurement'))
tau_options = "-optKeepFiles"
self.assertCommandReturnValue(0, EDIT_COMMAND, ['profile', '--extra-tau-options=%s' % tau_options])
with self.assertRaises(IncompatibleRecordError):
self.assertNotCommandReturnValue(0, EDIT_COMMAND, ['profile', '--force-tau-options=%s' % tau_options])
meas = Measurement.controller(PROJECT_STORAGE).one({'name': 'profile'})
self.assertIsNotNone(meas)
def test_export_otf2(self):
self.reset_project_storage(['--mpi', '--trace', 'otf2', '--profile', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['trace'], 'otf2')
self.assertEqual(meas['profile'], 'none')
self.assertManagedBuild(0, MPI_CC, [], 'mpi_hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd, ['mpirun', '-np', '4', './a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.tgz'
self.assertTrue(os.path.exists(export_file))
def test_export_slog2(self):
self.reset_project_storage(['--trace', 'slog2', '--profile', 'none'])
expr = Project.selected().experiment()
meas = expr.populate('measurement')
self.assertEqual(meas['trace'], 'slog2')
self.assertEqual(meas['profile'], 'none')
self.assertManagedBuild(0, CC, [], 'hello.c')
self.assertCommandReturnValue(EXIT_SUCCESS, trial_create_cmd,
['./a.out'])
self.assertCommandReturnValue(EXIT_SUCCESS, trial_export_cmd, [])
export_file = expr['name'] + '.trial0.slog2'
self.assertTrue(os.path.exists(export_file))
def main(self, argv):
args = self._parse_args(argv)
trial_numbers = []
for num in getattr(args, 'trial_numbers', []):
try:
trial_numbers.append(int(num))
except ValueError:
self.parser.error("Invalid trial number: %s" % num)
expr = Project.selected().experiment()
for trial in expr.trials(trial_numbers):
trial.export(args.destination)
return EXIT_SUCCESS
def main(self, argv):
args = self._parse_args(argv)
trial_numbers = []
for num in getattr(args, 'trial_numbers', []):
try:
trial_numbers.append(int(num))
except ValueError:
self.parser.error("Invalid trial number: %s" % num)
expr = Project.selected().experiment()
for trial in expr.trials(trial_numbers):
trial.export(args.destination)
return EXIT_SUCCESS
def test_set_tau_forced_extra_options_none(self):
self.reset_project_storage()
expr = Project.selected().experiment()
self.assertFalse('extra-tau-options' in expr.populate('measurement'))
self.assertFalse('forced-tau-options' in expr.populate('measurement'))
tau_options = "none"
self.assertCommandReturnValue(0, EDIT_COMMAND, ['profile', '--extra-tau-options=%s' % tau_options])
self.assertCommandReturnValue(0, EDIT_COMMAND, ['profile', '--force-tau-options=%s' % tau_options])
meas = Measurement.controller(PROJECT_STORAGE).one({'name': 'profile'})
self.assertIsNotNone(meas)
self.assertNotIn('extra_tau_options', meas)
self.assertNotIn('force_tau_options', meas)
def main(self, argv):
"""Command program entry point.
Args:
argv (list): Command line arguments.
Returns:
int: Process return code: non-zero if a problem occurred, 0 otherwise
"""
args = self._parse_args(argv)
style_args = ['--' + args.style] if hasattr(args, 'style') else []
levels = arguments.parse_storage_flag(args)
keys = getattr(args, 'keys', [])
single = (len(keys) == 1 and len(levels) == 1)
if single:
proj_name = keys[0]
self.title_fmt = "Project Configuration (%(storage_path)s)"
target_list_cmd.title_fmt = "Targets in project '%s'" % proj_name
application_list_cmd.title_fmt = "Applications in project '%s'" % proj_name
measurement_list_cmd.title_fmt = "Measurements in project '%s'" % proj_name
experiment_list_cmd.title_fmt = "Experiments in project '%s'" % proj_name
retval = super(ProjectListCommand, self).main(argv)
if single:
storage = levels[0]
ctrl = Project.controller(storage)
proj = ctrl.one({'name': keys[0]})
for cmd, prop in ((target_list_cmd, 'targets'),
(application_list_cmd, 'applications'),
(measurement_list_cmd, 'measurements'),
(experiment_list_cmd, 'experiments')):
primary_key = proj.attributes[prop]['collection'].key_attribute
records = proj.populate(prop)
if records:
cmd.main([record[primary_key] for record in records] + style_args)
else:
label = util.color_text('%s: No %s' % (proj['name'], prop), color='red', attrs=['bold'])
print "%s. Use `%s` to view available %s.\n" % (label, cmd, prop)
if proj.get('force_tau_options', False):
self.logger.warning("Project '%s' will add '%s' to TAU_OPTIONS without error checking.",
proj['name'], ' '.join(proj['force_tau_options']))
try:
expr = proj.experiment()
except ExperimentSelectionError:
print (util.color_text('No selected experiment: ', 'red') +
'Use `%s` to create or select an experiment.' % select_cmd)
else:
print util.color_text("Selected Experiment: ", 'cyan') + expr['name']
return retval
def is_compatible(cmd):
"""Check if this subcommand can work with the given command.
Args:
cmd (str): A command from the command line, e.g. sys.argv[1].
Returns:
bool: True if this subcommand is compatible with `cmd`.
"""
uses_python = Project.selected().experiment().populate()['application'].get_or_default('python')
if uses_python and 'python' in cmd:
return True
return bool(util.which(cmd))
def main(self, argv):
from taucmdr.cli.commands.project.list import COMMAND as project_list
args = self._parse_args(argv)
tar_ctrl = Target.controller(PROJECT_STORAGE)
app_ctrl = Application.controller(PROJECT_STORAGE)
meas_ctrl = Measurement.controller(PROJECT_STORAGE)
proj_ctrl = Project.controller()
project_name = args.name
project = proj_ctrl.one({'name': project_name})
if not project:
self.parser.error("'%s' is not a project name. Type `%s` to see valid names." %
(project_name, project_list.command))
updates = dict(project.element)
updates['name'] = getattr(args, 'new_name', project_name)
targets = set(project['targets'])
applications = set(project['applications'])
measurements = set(project['measurements'])
added = self._parse_add_args(args, tar_ctrl, app_ctrl, meas_ctrl, targets, applications, measurements)
removed = self._parse_remove_args(args, tar_ctrl, app_ctrl, meas_ctrl, targets, applications, measurements)
updates['targets'] = list(targets)
updates['applications'] = list(applications)
updates['measurements'] = list(measurements)
try:
force_tau_options = args.force_tau_options
except AttributeError:
pass
else:
# Unset force_tau_options if it was already set and --force-tau-options=none
if updates.pop('force_tau_options', False) and [i.lower().strip() for i in force_tau_options] == ['none']:
proj_ctrl.unset(['force_tau_options'], {'name': project_name})
self.logger.info("Removed 'force-tau-options' from project configuration '%s'.", project_name)
else:
updates['force_tau_options'] = force_tau_options
self.logger.info("Added 'force-tau-options' to project configuration '%s'.", project_name)
proj_ctrl.update(updates, {'name': project_name})
for model in added:
self.logger.info("Added %s '%s' to project configuration '%s'.",
model.name.lower(), model[model.key_attribute], project_name)
for model in removed:
self.logger.info("Removed %s '%s' from project configuration '%s'.",
model.name.lower(), model[model.key_attribute], project_name)
return EXIT_SUCCESS
def test_set_tau_force_options(self):
self.reset_project_storage()
proj_ctrl = Project.controller()
# Check that 'force-tau-options' is unset in the new project configuration
proj1 = proj_ctrl.one({'name': 'proj1'})
self.assertFalse('force-tau-options' in proj1)
# Test --force-tau-options
tau_options = "-optVerbose -optNoCompInst"
argv = ['proj1', '--force-tau-options=%s' % tau_options]
self.assertCommandReturnValue(0, edit.COMMAND, argv)
# Check that 'force-tau-options' is now a list containing the expected options in the project record
proj1 = proj_ctrl.one({'name': 'proj1'})
self.assertIsNotNone(proj1)
print proj1
self.assertListEqual(proj1['force_tau_options'], [tau_options])
def assertInLastTrialData(self, value, data_type='tau'):
from taucmdr.model.project import Project
trial = Project.selected().experiment().trials()
data_files = trial[0].get_data_files()
if data_type == 'tau':
data = []
for profile_file in glob.glob(os.path.join(data_files['tau'], 'profile.*.*.*')):
with open(profile_file) as fin:
buff = fin.read()
if value in buff:
return
data.append(buff)
else:
raise NotImplementedError
self.fail("'%s' not found in '%s'" % (value, '\n'.join(data)))
def main(self, argv):
"""Program entry point.
Args:
argv (list): Command line arguments.
Returns:
int: Process return code: non-zero if a problem occurred, 0 otherwise
"""
args = self._parse_args(argv)
cmd = args.command
cmd_args = args.options
log_level = getattr(args, 'verbose', getattr(args, 'quiet', logger.LOG_LEVEL))
logger.set_log_level(log_level)
LOGGER.debug('Arguments: %s', args)
LOGGER.debug('Verbosity level: %s', logger.LOG_LEVEL)
# Try to execute as a TAU command
try:
return cli.execute_command([cmd], cmd_args)
except UnknownCommandError:
pass
# Check shortcuts
shortcut = None
if build_command.is_compatible(cmd):
shortcut = ['build']
cmd_args = [cmd] + cmd_args
elif trial_create_command.is_compatible(cmd):
shortcut = ['trial', 'create']
cmd_args = [cmd] + cmd_args
elif 'show'.startswith(cmd):
shortcut = ['trial', 'show']
elif 'metrics'.startswith(cmd):
expr = Project.selected().experiment()
targ_name = expr.populate('target')['name']
shortcut = ['target', 'metrics']
cmd_args.insert(0, targ_name)
if shortcut:
LOGGER.debug('Trying shortcut: %s', shortcut)
return cli.execute_command(shortcut, cmd_args)
else:
LOGGER.debug('No shortcut found for %r', cmd)
# Not sure what to do at this point, so advise the user and exit
LOGGER.info("Unknown command. Calling `%s help %s` to get advice.", TAUCMDR_SCRIPT, cmd)
return cli.execute_command(['help'], [cmd])
def main(self, argv):
args = self._parse_args(argv)
proj_ctrl = Project.controller()
trial_ctrl = Trial.controller(proj_ctrl.storage)
proj = proj_ctrl.selected()
expr = proj.experiment()
try:
number = int(args.number)
except ValueError:
self.parser.error("Invalid trial number: %s" % args.number)
fields = {'experiment': expr.eid, 'number': number}
if not trial_ctrl.exists(fields):
self.parser.error("No trial number %s in the current experiment. "
"See `trial list` to see all trial numbers." % number)
trial_ctrl.delete(fields)
self.logger.info('Deleted trial %s', number)
return EXIT_SUCCESS
def main(self, argv):
args = self._parse_args(argv)
store = arguments.parse_storage_flag(args)[0]
data = {attr: getattr(args, attr) for attr in self.model.attributes if hasattr(args, attr)}
proj_ctrl = Project.controller()
for model in Target, Application, Measurement:
attr = model.name.lower()
if attr in data:
record = model.controller(proj_ctrl.storage).one({'name': data[attr]})
data[attr] = record.eid
key_attr = self.model.key_attribute
try:
data[key_attr] = args.new_key
except AttributeError:
pass
key = getattr(args, key_attr)
return self._update_record(store, data, key)
def main(self, argv):
args = self._parse_args(argv)
proj_ctrl = Project.controller()
proj = proj_ctrl.selected()
targ = Target.controller(proj_ctrl.storage).one({'name': args.target})
if targ is None:
self.parser.error("A target with name %s does not exist." %args.target)
app = Application.controller(proj_ctrl.storage).one({'name': args.application})
if app is None:
self.parser.error("An application with name %s does not exist." %args.application)
meas = Measurement.controller(proj_ctrl.storage).one({'name': args.measurement})
if meas is None:
self.parser.error("A measurement with name %s does not exist." %args.measurement)
data = {'name': args.name,
'project': proj.eid,
'target': targ.eid,
'application': app.eid,
'measurement': meas.eid}
return self._create_record(PROJECT_STORAGE, data)
def main(self, argv):
args = self._parse_args(argv)
subargs = ['--' + args.style]
proj_ctrl = Project.controller()
print
try:
proj = proj_ctrl.selected()
except ProjectSelectionError as err:
project_count = proj_ctrl.count()
if project_count > 0:
project_list_cmd.main(subargs)
err.value = "No project configuration selected. There are %d configurations available." % project_count
else:
from taucmdr.cli.commands.project.create import COMMAND as project_create_cmd
err.value = "No project configurations exist."
err.hints = ['Use `%s` to create a new project configuration.' % project_create_cmd]
raise err
else:
project_list_cmd.main(subargs + [proj['name']])
print
return EXIT_SUCCESS
def select(cls, name):
"""Changes the selected experiment in the current project.
Raises:
ExperimentSelectionError: No experiment with the given name in the currently selected project.
Args:
name (str): Name of the experiment to select.
"""
proj_ctrl = Project.controller()
proj = proj_ctrl.selected()
expr_ctrl = cls.controller()
data = {"name": name, "project": proj.eid}
matching = expr_ctrl.search(data)
if not matching:
raise ExperimentSelectionError("There is no experiment named '%s' in project '%s'." % (name, proj['name']))
elif len(matching) > 1:
raise InternalError("More than one experiment with data %r exists!" % data)
else:
expr = matching[0]
proj_ctrl.select(proj, expr)
def dashboard_format(self, records):
"""Format modeled records in dashboard format.
Args:
records: Modeled records to format.
Returns:
str: Record data in dashboard format.
"""
self.logger.debug("Dashboard format")
title = util.hline(self.title_fmt % {'model_name': records[0].name.capitalize(),
'storage_path': records[0].storage}, 'cyan')
expr = Project.selected().experiment()
subtitle = util.color_text("Selected experiment: ", 'cyan') + expr['name']
header_row = [col['header'] for col in self.dashboard_columns]
rows = [header_row]
for record in records:
populated = record.populate()
row = []
for col in self.dashboard_columns:
if 'value' in col:
try:
cell = populated[col['value']]
except KeyError:
cell = 'N/A'
elif 'yesno' in col:
cell = 'Yes' if populated.get(col['yesno'], False) else 'No'
elif 'function' in col:
cell = col['function'](populated)
else:
raise InternalError("Invalid column definition: %s" % col)
row.append(cell)
rows.append(row)
table = Texttable(logger.LINE_WIDTH)
table.set_cols_align([col.get('align', 'c') for col in self.dashboard_columns])
table.add_rows(rows)
return [title, table.draw(), '', subtitle, '']
def main(self, argv):
from taucmdr.cli.commands.project.list import COMMAND as project_list
args = self._parse_args(argv)
tar_ctrl = Target.controller(PROJECT_STORAGE)
app_ctrl = Application.controller(PROJECT_STORAGE)
meas_ctrl = Measurement.controller(PROJECT_STORAGE)
proj_ctrl = Project.controller()
project_name = args.name
project = proj_ctrl.one({'name': project_name})
if not project:
self.parser.error("'%s' is not a project name. Type `%s` to see valid names." %
(project_name, project_list.command))
updates = dict(project)
updates['name'] = getattr(args, 'new_name', project_name)
targets = set(project['targets'])
applications = set(project['applications'])
measurements = set(project['measurements'])
added = self._parse_add_args(args, tar_ctrl, app_ctrl, meas_ctrl, targets, applications, measurements)
removed = self._parse_remove_args(args, tar_ctrl, app_ctrl, meas_ctrl, targets, applications, measurements)
updates['targets'] = list(targets)
updates['applications'] = list(applications)
updates['measurements'] = list(measurements)
proj_ctrl.update(updates, {'name': project_name})
for model in added:
self.logger.info("Added %s '%s' to project configuration '%s'.",
model.name.lower(), model[model.key_attribute], project_name)
for model in removed:
self.logger.info("Removed %s '%s' from project configuration '%s'.",
model.name.lower(), model[model.key_attribute], project_name)
return EXIT_SUCCESS
def main(self, argv):
self._parse_args(argv)
launcher_cmd, application_cmds = Trial.parse_launcher_cmd(argv)
self.logger.debug("Launcher command: %s", launcher_cmd)
self.logger.debug("Application commands: %s", application_cmds)
return Project.selected().experiment().managed_run(launcher_cmd, application_cmds)
def main(self, argv):
args = self._parse_args(argv)
expr = Project.selected().experiment()
return expr.managed_build(args.cmd, args.cmd_args)
def _get_papi_installation(self):
expr = Project.selected().experiment()
return expr.populate('target').get_installation('papi')
