def find_command(cmd):
"""Import the command module and return its COMMAND member.
Args:
cmd (list): List of strings identifying the command, i.e. from :any:`_command_as_list`.
Raises:
UnknownCommandError: `cmd` is invalid.
AmbiguousCommandError: `cmd` is ambiguous.
Returns:
AbstractCommand: Command object for the subcommand.
"""
if cmd:
root = '.'.join([COMMANDS_PACKAGE_NAME] + cmd)
else:
root = COMMANDS_PACKAGE_NAME
try:
return _get_commands(root)['__module__'].COMMAND
except KeyError:
LOGGER.debug('%r not recognized as a TAU command', cmd)
resolved = _resolve(cmd, cmd, _COMMANDS[SCRIPT_COMMAND])
LOGGER.debug('Resolved ambiguous command %r to %r', cmd, resolved)
return find_command(resolved)
except AttributeError:
raise InternalError("'COMMAND' undefined in %r" % cmd)
def _copy_record(self, store, updates, key):
ctrl = self.model.controller(store)
key_attr = self.model.key_attribute
matching = ctrl.search({key_attr: key})
if not matching:
self.parser.error(
"No %s-level %s with %s='%s'." %
(ctrl.storage.name, self.model_name, key_attr, key))
elif len(matching) > 1:
raise InternalError(
"More than one %s-level %s with %s='%s' exists!" %
(ctrl.storage.name, self.model_name, key_attr, key))
else:
found = matching[0]
data = dict(found)
data.pop('experiment', None)
data.pop('experiments', None)
data.update(updates)
key_attr = self.model.key_attribute
key = data[key_attr]
try:
ctrl.create(data)
except UniqueAttributeError:
self.parser.error("A %s with %s='%s' already exists" %
(self.model_name, key_attr, key))
self.logger.info("Created a new %s-level %s: '%s'.", ctrl.storage.name,
self.model_name, key)
return EXIT_SUCCESS
def get_parser(prog=None, usage=None, description=None, epilog=None):
"""Builds an argument parser.
The returned argument parser accepts no arguments.
Use :any:`argparse.ArgumentParser.add_argument` to add arguments.
Args:
prog (str): Name of the program.
usage (str): Description of the program's usage.
description (str): Text to display before the argument help.
epilog (str): Text to display after the argument help.
Returns:
MutableArgumentGroupParser: The customized argument parser object.
"""
try:
formatter = getattr(sys.modules[__name__],
USAGE_FORMAT.capitalize() + 'HelpFormatter')
except AttributeError:
raise InternalError("Invalid USAGE_FORMAT: %s" % USAGE_FORMAT)
return MutableArgumentGroupParser(prog=prog,
usage=usage,
description=description,
epilog=epilog,
formatter_class=formatter)
def _create_dl_subprocess(abs_cmd, src, dest, timeout):
if "curl" in os.path.basename(abs_cmd):
size_cmd = [abs_cmd, '-sI', src, '--location', '--max-time', str(timeout)]
get_cmd = [abs_cmd, '-s', '-L', src, '-o', dest, '--connect-timeout', str(timeout)]
elif "wget" in os.path.basename(abs_cmd):
size_cmd = [abs_cmd, src, '--spider', '--server-response', '--timeout=%d' % timeout, '--tries=1']
get_cmd = [abs_cmd, '-q', src, '-O', dest, '--timeout=%d' % timeout]
else:
raise InternalError("Invalid command parameter: %s" % abs_cmd)
try:
proc_output = get_command_output(size_cmd)
except subprocess.CalledProcessError as err:
return err.returncode
_heavy_debug(proc_output)
try:
file_size = int(proc_output.partition('Content-Length')[2].split()[1])
except (ValueError, IndexError):
LOGGER.warning("Invalid response while retrieving download file size")
file_size = -1
with ProgressIndicator(file_size) as progress_bar:
with open(os.devnull, 'wb') as devnull:
proc = subprocess.Popen(get_cmd, stdout=devnull, stderr=subprocess.STDOUT)
while proc.poll() is None:
try:
current_size = os.stat(dest).st_size
except OSError:
pass
else:
progress_bar.update(current_size)
time.sleep(0.1)
proc.wait()
retval = proc.returncode
LOGGER.debug("%s returned %d", get_cmd, retval)
return retval
def _associate(self, record, foreign_model, affected, via):
"""Associates a record with another record.
Args:
record (Record): Record to associate.
foreign_model (Model): Foreign record's data model.
affected (list): Identifiers for the records that will be updated to associate with `record`.
via (str): The name of the associated foreign attribute.
"""
_heavy_debug("Adding %s to '%s' in %s(eids=%s)", record.eid, via,
foreign_model.name, affected)
if not isinstance(affected, list):
affected = [affected]
with self.storage as database:
for key in affected:
foreign_record = database.get(key,
table_name=foreign_model.name)
if not foreign_record:
raise ModelError(foreign_model,
"No record with ID '%s'" % key)
if 'model' in foreign_model.attributes[via]:
updated = record.eid
elif 'collection' in foreign_model.attributes[via]:
updated = list(set(foreign_record[via] + [record.eid]))
else:
raise InternalError(
"%s.%s has neither 'model' nor 'collection'" %
(foreign_model.name, via))
foreign_model.controller(database).update({via: updated}, key)
def _format_long_item(self, key, val):
attrs = self.model.attributes[key]
if 'collection' in attrs:
foreign_model = attrs['collection']
foreign_keys = []
for foreign_record in val:
try:
foreign_keys.append(
str(foreign_record[foreign_model.key_attribute]))
except (AttributeError, ModelError):
foreign_keys.append(str(foreign_record))
val = ', '.join(foreign_keys)
elif 'model' in attrs:
foreign_model = attrs['model']
try:
val = str(val[foreign_model.key_attribute])
except (AttributeError, ModelError):
val = str(val)
elif 'type' in attrs:
if attrs['type'] == 'boolean':
val = str(bool(val))
elif attrs['type'] == 'array':
val = ', '.join(str(x) for x in val)
elif attrs['type'] != 'string':
val = str(val)
else:
raise InternalError("Attribute has no type: %s, %s" % (attrs, val))
description = attrs.get('description', 'No description')
description = description[0].upper() + description[1:] + "."
flags = ', '.join(
flag
for flag in attrs.get('argparse', {'flags': ('N/A', )})['flags'])
return [key, val, flags, description]
def _compiler_info(self):
command = os.path.basename(self['path'])
role = Knowledgebase.find_role(self['role'])
family = role.kbase.families[self['family']]
info_list = Knowledgebase.find_compiler(command, family, role)
if len(info_list) != 1:
raise InternalError("Zero or more than one known compilers match '%s'" % self)
return info_list[0]
def select(self, project, experiment=None):
self.storage['selected_project'] = project.eid
if experiment is not None:
for attr in 'target', 'application', 'measurement':
if experiment[attr] not in project[attr + 's']:
raise InternalError(
"Experiment contains %s not in project" % attr)
self.update({'experiment': experiment.eid}, project.eid)
experiment.configure()
def get_data_files(self):
"""Return paths to the trial's data files or directories maped by data type.
Post-process trial data if necessary and return a dictionary mapping the types of data produced
by this trial to paths to related data files or directories. The paths should be suitable for
passing on a command line to one of the known data analysis tools. For example, a trial producing
SLOG2 traces and TAU profiles would return ``{"slog2": "/path/to/tau.slog2", "tau": "/path/to/directory/"}``.
Returns:
dict: Keys are strings indicating the data type; values are filesystem paths.
"""
expr = self.populate('experiment')
if self.get('data_size', 0) <= 0:
raise ConfigurationError(
"Trial %s of experiment '%s' has no data" %
(self['number'], expr['name']))
meas = self.populate('experiment').populate('measurement')
profile_fmt = meas.get('profile', 'none')
trace_fmt = meas.get('trace', 'none')
if trace_fmt == 'slog2':
self._postprocess_slog2()
data = {}
if profile_fmt == 'tau':
data[profile_fmt] = self.prefix
elif profile_fmt == 'merged':
data[profile_fmt] = os.path.join(self.prefix, 'tauprofile.xml')
elif profile_fmt == 'cubex':
data[profile_fmt] = os.path.join(self.prefix, 'profile.cubex')
elif profile_fmt != 'none':
raise InternalError("Unhandled profile format '%s'" % profile_fmt)
trace_fmt = meas.get('trace', 'none')
if trace_fmt == 'slog2':
data[trace_fmt] = os.path.join(self.prefix, 'tau.slog2')
elif trace_fmt == 'otf2':
data[trace_fmt] = os.path.join(self.prefix, 'traces.otf2')
elif trace_fmt != 'none':
raise InternalError("Unhandled trace format '%s'" % trace_fmt)
return data
def _copy_record(self, store, updates, key):
ctrl = self.model.controller(store)
key_attr = self.model.key_attribute
matching = ctrl.search({key_attr: key})
if not matching:
self.parser.error(
"No %s-level %s with %s='%s'." %
(ctrl.storage.name, self.model_name, key_attr, key))
elif len(matching) > 1:
raise InternalError(
"More than one %s-level %s with %s='%s' exists!" %
(ctrl.storage.name, self.model_name, key_attr, key))
else:
found = matching[0]
data = dict(found)
data.update(updates)
return self._create_record(store, data)
def dashboard_format(self, records):
"""Format modeled records in dashboard format.
Args:
records: Modeled records to format.
Returns:
str: Record data in dashboard format.
"""
self.logger.debug("Dashboard format")
title = util.hline(
self.title_fmt % {
'model_name': records[0].name.capitalize(),
'storage_path': records[0].storage
}, 'cyan')
expr = Project.selected().experiment()
subtitle = util.color_text("Selected experiment: ",
'cyan') + expr['name']
header_row = [col['header'] for col in self.dashboard_columns]
rows = [header_row]
for record in records:
populated = record.populate()
row = []
for col in self.dashboard_columns:
if 'value' in col:
try:
cell = populated[col['value']]
except KeyError:
cell = 'N/A'
elif 'yesno' in col:
cell = 'Yes' if populated.get(col['yesno'],
False) else 'No'
elif 'function' in col:
cell = col['function'](populated)
else:
raise InternalError("Invalid column definition: %s" % col)
row.append(cell)
rows.append(row)
table = Texttable(logger.LINE_WIDTH)
table.set_cols_align(
[col.get('align', 'c') for col in self.dashboard_columns])
table.add_rows(rows)
return [title, table.draw(), '', subtitle, '']
def export(self, dest):
"""Export experiment trial data.
Args:
dest (str): Path to directory to contain exported data.
Raises:
ConfigurationError: This trial has no data.
"""
expr = self.populate('experiment')
if self.get('data_size', 0) <= 0:
raise ConfigurationError(
"Trial %s of experiment '%s' has no data" %
(self['number'], expr['name']))
data = self.get_data_files()
stem = '%s.trial%d' % (expr['name'], self['number'])
for fmt, path in data.iteritems():
if fmt == 'tau':
export_file = os.path.join(dest, stem + '.ppk')
tau = TauInstallation.get_minimal()
tau.create_ppk_file(export_file, path)
elif fmt == 'merged':
export_file = os.path.join(dest, stem + '.xml.gz')
util.create_archive('gz', export_file, [path])
elif fmt == 'cubex':
export_file = os.path.join(dest, stem + '.cubex')
LOGGER.info("Writing '%s'...", export_file)
util.copy_file(path, export_file)
elif fmt == 'slog2':
export_file = os.path.join(dest, stem + '.slog2')
LOGGER.info("Writing '%s'...", export_file)
util.copy_file(path, export_file)
elif fmt == 'otf2':
export_file = os.path.join(dest, stem + '.tgz')
expr_dir, trial_dir = os.path.split(os.path.dirname(path))
items = [
os.path.join(trial_dir, item)
for item in 'traces', 'traces.def', 'traces.otf2'
]
util.create_archive('tgz', export_file, items, expr_dir)
elif fmt != 'none':
raise InternalError("Unhandled data file format '%s'" % fmt)
def __new__(mcs, name, bases, dct):
if dct['__module__'] != __name__:
# Each Model subclass has its own relationships
dct.update({'associations': dict(), 'references': set()})
# The default model name is the class name
if dct.get('name', None) is None:
dct['name'] = name
# Model subclasses must define __attributes__ as a callable.
# We make the callable a staticmethod to prevent method binding.
try:
dct['__attributes__'] = staticmethod(dct['__attributes__'])
except KeyError:
raise InternalError("Model class %s does not define '__attributes__'" % name)
# Replace attributes with a callable property (defined below). This is to guarantee
# that model attributes won't be constructed until all Model subclasses have been constructed.
dct['attributes'] = ModelMeta.attributes
# Replace key_attribute with a callable property (defined below). This is to set
# the key_attribute member after the model attributes have been constructed.
dct['key_attribute'] = ModelMeta.key_attribute
return type.__new__(mcs, name, bases, dct)
def get_all_commands(package_name=COMMANDS_PACKAGE_NAME):
"""Builds a list of all commands and subcommands.
Args:
package_name (str): A dot-separated string naming the module to search for cli.
Returns:
list: List of modules corresponding to all commands and subcommands.
"""
all_commands = []
commands = sorted((i for i in _get_commands(package_name).iteritems()
if i[0] != '__module__'))
for _, topcmd in commands:
for _, mod in topcmd.iteritems():
if isinstance(mod, dict):
all_commands.append(mod['__module__'].__name__)
elif isinstance(mod, ModuleType):
all_commands.append(mod.__name__)
else:
raise InternalError("%s is an invalid module." % mod)
return all_commands
def select(cls, name):
"""Changes the selected experiment in the current project.
Raises:
ExperimentSelectionError: No experiment with the given name in the currently selected project.
Args:
name (str): Name of the experiment to select.
"""
proj_ctrl = Project.controller()
proj = proj_ctrl.selected()
expr_ctrl = cls.controller()
data = {"name": name, "project": proj.eid}
matching = expr_ctrl.search(data)
if not matching:
raise ExperimentSelectionError("There is no experiment named '%s' in project '%s'." % (name, proj['name']))
elif len(matching) > 1:
raise InternalError("More than one experiment with data %r exists!" % data)
else:
expr = matching[0]
proj_ctrl.select(proj, expr)
def create_archive(fmt, dest, items, cwd=None, show_progress=True):
"""Creates a new archive file in the specified format.
Args:
fmt (str): Archive fmt, e.g. 'zip' or 'tgz'.
dest (str): Path to the archive file that will be created.
items (list): Items (i.e. files or folders) to add to the archive.
cwd (str): Current working directory while creating the archive.
"""
if cwd:
oldcwd = os.getcwd()
os.chdir(cwd)
if show_progress:
LOGGER.info("Writing '%s'...", dest)
context = ProgressIndicator
else:
context = _null_context
with context(""):
try:
if fmt == 'zip':
with ZipFile(dest, 'w') as archive:
archive.comment = "Created by TAU Commander"
for item in items:
archive.write(item)
elif fmt in ('tar', 'tgz', 'tar.bz2'):
mode_map = {'tar': 'w', 'tgz': 'w:gz', 'tar.bz2': 'w:bz2'}
with tarfile.open(dest, mode_map[fmt]) as archive:
for item in items:
archive.add(item)
elif fmt == 'gz':
with open(items[0], 'rb') as fin, gzip.open(dest,
'wb') as fout:
shutil.copyfileobj(fin, fout)
else:
raise InternalError("Invalid archive format: %s" % fmt)
finally:
if cwd:
os.chdir(oldcwd)
def format_help(self):
try:
func = getattr(self, '_format_help_' + USAGE_FORMAT.lower())
except AttributeError:
raise InternalError("Invalid USAGE_FORMAT: %s" % USAGE_FORMAT)
return func()
def _safe_execute(self, cmd, argv):
retval = cmd.main(argv)
if retval != EXIT_SUCCESS:
raise InternalError("return code %s: %s %s" % (retval, cmd, ' '.join(argv)))
def create(self, data):
if self.storage is not PROJECT_STORAGE:
raise InternalError(
"Projects may only be created in project-level storage")
return super(ProjectController, self).create(data)
def __delitem__(self, key):
raise InternalError(
"Use controller(storage).update() to alter records")