def verify(self):
"""Check if the installation at :any:`installation_prefix` is valid.
A valid installation provides all expected files and commands.
Subclasses may wish to perform additional checks.
Raises:
SoftwarePackageError: Describs why the installation is invalid.
"""
LOGGER.debug("Verifying %s installation at '%s'", self.title, self.install_prefix)
if not os.path.exists(self.install_prefix):
raise SoftwarePackageError("'%s' does not exist" % self.install_prefix)
for cmd in self.verify_commands:
path = os.path.join(self.bin_path, cmd)
if not os.path.exists(path):
raise SoftwarePackageError("'%s' is missing" % path)
if not os.access(path, os.X_OK):
raise SoftwarePackageError("'%s' exists but is not executable" % path)
for lib in self.verify_libraries:
path = os.path.join(self.lib_path, lib)
if not util.path_accessible(path):
# Some systems (e.g. SuSE) append the machine bitwidth to the library path
path = os.path.join(self.lib_path+'64', lib)
if not util.path_accessible(path):
raise SoftwarePackageError("'%s' is not accessible" % path)
for header in self.verify_headers:
path = os.path.join(self.include_path, header)
if not util.path_accessible(path):
raise SoftwarePackageError("'%s' is not accessible" % path)
LOGGER.debug("%s installation at '%s' is valid", self.name, self.install_prefix)
def verify(self):
"""Check if the installation at :any:`installation_prefix` is valid.
A valid installation provides all expected files and commands.
Subclasses may wish to perform additional checks.
Raises:
SoftwarePackageError: Describs why the installation is invalid.
"""
LOGGER.debug("Verifying %s installation at '%s'", self.title,
self.install_prefix)
if not os.path.exists(self.install_prefix):
raise SoftwarePackageError("'%s' does not exist" %
self.install_prefix)
for cmd in self.verify_commands:
path = os.path.join(self.bin_path, cmd)
if not os.path.exists(path):
raise SoftwarePackageError("'%s' is missing" % path)
if not os.access(path, os.X_OK):
raise SoftwarePackageError(
"'%s' exists but is not executable" % path)
for lib in self.verify_libraries:
path = os.path.join(self.lib_path, lib)
if not util.path_accessible(path):
# Some systems (e.g. SuSE) append the machine bitwidth to the library path
path = os.path.join(self.lib_path + '64', lib)
if not util.path_accessible(path):
raise SoftwarePackageError("'%s' is not accessible" % path)
for header in self.verify_headers:
path = os.path.join(self.include_path, header)
if not util.path_accessible(path):
raise SoftwarePackageError("'%s' is not accessible" % path)
LOGGER.debug("%s installation at '%s' is valid", self.name,
self.install_prefix)
def __call__(self, parser, namespace, value, unused_option_string=None):
"""Sets the `self.dest` attribute in `namespace` to the parsed value of `value`.
If `value` parses to a boolean True value then the attribute value is 'download'.
If `value` parses to a boolean False value then the attribute value is ``None``.
Otherwise the attribute value is the value of `value`.
Args:
parser (str): Argument parser object this group belongs to.
namespace (object): Namespace to receive parsed value via setattr.
value (str): Value parsed from the command line.
"""
try:
value_as_bool = util.parse_bool(
value, additional_true=['download', 'download-tr6', 'nightly'])
except TypeError:
if not util.is_url(value):
value = os.path.abspath(os.path.expanduser(value))
if not (os.path.isdir(value) or util.path_accessible(value)):
raise argparse.ArgumentError(
self,
"Keyword, valid path, or URL required: %s" % value)
else:
value = value.lower() if value_as_bool else None
setattr(namespace, self.dest, value)
def _separate_launcher_cmd(cls, cmd):
"""Separate the launcher command and it's arguments from the application command(s) and arguments.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, Remainder of command line)
Raises:
ConfigurationError: No application config files or executables found after a recognized launcher command.
"""
# If '--' appears in the command then everything before it is a launcher + args
# and everything after is the application + args
try:
idx = cmd.index('--')
except ValueError:
pass
else:
return cmd[:idx], cmd[idx + 1:]
cmd0 = cmd[0]
for launcher, appfile_flags in PROGRAM_LAUNCHERS.iteritems():
if launcher not in cmd0:
continue
# No '--' to indicate start of application, so look for first executable
for idx, exe in enumerate(cmd[1:], 1):
if util.which(exe):
return cmd[:idx], cmd[idx:]
# No exectuables, so look for application config file
if appfile_flags:
for i, arg in enumerate(cmd[1:], 1):
try:
arg, appfile = arg.split('=')
except ValueError:
try:
appfile = cmd[i + 1]
except IndexError:
# Reached the end of the command line without finding an application config file
break
if arg in appfile_flags and util.path_accessible(appfile):
return cmd, []
raise ConfigurationError(
("TAU is having trouble parsing the command line: no executable "
"commands or %s application files were found after "
"the launcher command '%s'") % (cmd0, cmd0),
"Check that the command is correct. Does it work without TAU?",
("Use '--' to seperate '%s' and its arguments from the application "
"command, e.g. `mpirun -np 4 -- ./a.out -l hello`" % cmd0))
# No launcher command, just an application command
return [], cmd
def connect_database(self, *args, **kwargs):
"""Open the database for reading and writing."""
if self._database is None:
util.mkdirp(self.prefix)
dbfile = os.path.join(self.prefix, self.name + '.json')
try:
self._database = tinydb.TinyDB(dbfile, storage=_JsonFileStorage)
except IOError as err:
raise StorageError("Failed to access %s database '%s': %s" % (self.name, dbfile, err),
"Check that you have `write` access")
if not util.path_accessible(dbfile):
raise StorageError("Database file '%s' exists but cannot be read." % dbfile,
"Check that you have `read` access")
LOGGER.debug("Initialized %s database '%s'", self.name, dbfile)
def _separate_launcher_cmd(cls, cmd):
"""Separate the launcher command and it's arguments from the application command(s) and arguments.
Args:
cmd (list): Command line.
Returns:
tuple: (Launcher command, Remainder of command line)
Raises:
ConfigurationError: No application config files or executables found after a recognized launcher command.
"""
# If '--' appears in the command then everything before it is a launcher + args
# and everything after is the application + args
try:
idx = cmd.index('--')
except ValueError:
pass
else:
return cmd[:idx], cmd[idx+1:]
cmd0 = cmd[0]
for launcher, appfile_flags in PROGRAM_LAUNCHERS.iteritems():
if launcher not in cmd0:
continue
# No '--' to indicate start of application, so look for first executable
for idx, exe in enumerate(cmd[1:], 1):
if util.which(exe):
return cmd[:idx], cmd[idx:]
# No exectuables, so look for application config file
if appfile_flags:
for i, arg in enumerate(cmd[1:], 1):
try:
arg, appfile = arg.split('=')
except ValueError:
try:
appfile = cmd[i+1]
except IndexError:
# Reached the end of the command line without finding an application config file
break
if arg in appfile_flags and util.path_accessible(appfile):
return cmd, []
raise ConfigurationError(("TAU is having trouble parsing the command line: no executable "
"commands or %s application files were found after "
"the launcher command '%s'") % (cmd0, cmd0),
"Check that the command is correct. Does it work without TAU?",
("Use '--' to seperate '%s' and its arguments from the application "
"command, e.g. `mpirun -np 4 -- ./a.out -l hello`" % cmd0))
# No launcher command, just an application command
return [], cmd
def connect_database(self, *args, **kwargs):
"""Open the database for reading and writing."""
if self._database is None:
util.mkdirp(self.prefix)
dbfile = os.path.join(self.prefix, self.name + '.json')
try:
storage = CachingMiddleware(_JsonFileStorage)
storage.WRITE_CACHE_SIZE = 0
self._database = tinydb.TinyDB(dbfile, storage=storage)
except IOError as err:
raise StorageError("Failed to access %s database '%s': %s" % (self.name, dbfile, err),
"Check that you have `write` access")
if not util.path_accessible(dbfile):
raise StorageError("Database file '%s' exists but cannot be read." % dbfile,
"Check that you have `read` access")
LOGGER.debug("Initialized %s database '%s'", self.name, dbfile)
def tau_source_default():
""""Per Sameer's request, override managed TAU installation with an existing unmanaged TAU installation.
If a file named "override_tau_source" exists in the system-level storage prefix, use the contents of
that file as the default path for TAU. Otherwise use "download" as the default.
Returns:
str: Path to TAU or "download".
"""
try:
with open(os.path.join(SYSTEM_STORAGE.prefix, 'override_tau_source')) as fin:
path = fin.read()
except IOError:
return 'download'
path = path.strip()
if not (os.path.isdir(path) and util.path_accessible(path)):
LOGGER.warning("'%s' does not exist or is not accessable.")
return 'download'
return path
def tau_source_default():
""""Per Sameer's request, override managed TAU installation with an existing unmanaged TAU installation.
If a file named "override_tau_source" exists in the system-level storage prefix, use the contents of
that file as the default path for TAU. Otherwise use "download" as the default.
Returns:
str: Path to TAU or "download".
"""
try:
with open(os.path.join(SYSTEM_STORAGE.prefix,
'override_tau_source')) as fin:
path = fin.read()
except IOError:
return 'download'
path = path.strip()
if not (os.path.isdir(path) and util.path_accessible(path)):
LOGGER.warning("'%s' does not exist or is not accessable.")
return 'download'
return path
def __call__(self, parser, namespace, value, unused_option_string=None):
"""Sets the `self.dest` attribute in `namespace` to the parsed value of `value`.
If `value` parses to a boolean True value then the attribute value is 'download'.
If `value` parses to a boolean False value then the attribute value is ``None``.
Otherwise the attribute value is the value of `value`.
Args:
parser (str): Argument parser object this group belongs to.
namespace (object): Namespace to receive parsed value via setattr.
value (str): Value parsed from the command line.
"""
try:
value_as_bool = util.parse_bool(value, additional_true=['download', 'nightly'])
except TypeError:
if not util.is_url(value):
value = os.path.abspath(os.path.expanduser(value))
if not (os.path.isdir(value) or util.path_accessible(value)):
raise argparse.ArgumentError(self, "Keyword, valid path, or URL required: %s" % value)
else:
value = value.lower() if value_as_bool else None
setattr(namespace, self.dest, value)
def _parse_tau_makefile(self, args):
# Parsing a TAU Makefile is a really hairy operation, so let's lift the limits
# pylint: disable=too-many-statements,too-many-locals
makefile = args.forced_makefile
if not util.path_accessible(makefile):
self.parser.error("Invalid TAU makefile: %s" % makefile)
tau_arch_name = os.path.basename(
os.path.dirname(os.path.dirname(makefile)))
matches = [
arch for arch in TauMagic.all() if arch.name == tau_arch_name
]
if len(matches) == 1:
tau_arch = matches[0]
elif not matches:
raise ConfigurationError(
"TAU Makefile '%s' targets an unrecognized TAU architecture: %s"
% (makefile, tau_arch_name))
else:
for arch in matches:
if arch.architecture == HOST_ARCH and arch.operating_system == HOST_OS:
tau_arch = arch
break
else:
parts = [
"TAU Makefile '%s' targets an ambiguous TAU architecture: %s"
% (makefile, tau_arch_name), "It could be any of these:"
]
parts.extend([
" - %s on %s" %
(arch.operating_system.name, arch.architecture.name)
for arch in matches
])
raise ConfigurationError("\n".join(parts))
self.logger.info(
"Parsing TAU Makefile '%s' to populate command line arguments:",
makefile)
args.host_arch = tau_arch.architecture.name
self.logger.info(" --host-arch='%s'", args.host_arch)
args.host_os = tau_arch.operating_system.name
self.logger.info(" --host-os='%s'", args.host_os)
args.tau_source = os.path.abspath(
os.path.join(os.path.dirname(makefile), '..', '..'))
self.logger.info(" --taucmdr='%s'", args.tau_source)
with open(makefile, 'r') as fin:
compiler_parts = ("FULL_CC", "FULL_CXX", "TAU_F90")
package_parts = {
"BFDINCLUDE":
("binutils_source",
lambda x: os.path.dirname(shlex.split(x)[0].lstrip("-I"))),
"UNWIND_INC": ("libunwind_source",
lambda x: os.path.dirname(x.lstrip("-I"))),
"PAPIDIR": ("papi_source", os.path.abspath),
"PDTDIR": ("pdt_source", os.path.abspath),
"SCOREPDIR": ("scorep_source", os.path.abspath)
}
tau_r = ''
for line in fin:
if line.startswith('#'):
continue
try:
key, val = [x.strip() for x in line.split('=', 1)]
except ValueError:
continue
if key == 'TAU_R':
tau_r = val.split()[0]
elif key in compiler_parts:
path = util.which(val.strip().split()[0].replace(
'$(TAU_R)', tau_r))
if not path:
self.logger.warning(
"Failed to parse %s in TAU Makefile '%s'", key,
makefile)
continue
matching_info = Knowledgebase.find_compiler(
os.path.basename(path))
if matching_info:
if len(matching_info) > 1:
self.logger.warning(
"Ambiguous compiler '%s' in TAU Makefile '%s'",
path, makefile)
comp = InstalledCompiler(path, matching_info[0])
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'",
attr.lower().replace("_", "-"),
comp.absolute_path)
while comp.wrapped:
comp = comp.wrapped
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'",
attr.lower().replace("_", "-"),
comp.absolute_path)
elif key in package_parts:
attr, operator = package_parts[key]
path = val.strip()
if not path:
path = None
else:
path = operator(path)
if not os.path.exists(path):
self.logger.warning(
"'%s' referenced by TAU Makefile '%s' doesn't exist",
path, makefile)
continue
setattr(args, attr, path)
self.logger.info(" --%s='%s'",
attr.replace("_source", ""), path)
def _parse_tau_makefile(self, args):
# Parsing a TAU Makefile is a really hairy operation, so let's lift the limits
# pylint: disable=too-many-statements,too-many-locals
makefile = args.forced_makefile
if not util.path_accessible(makefile):
self.parser.error("Invalid TAU makefile: %s" % makefile)
tau_arch_name = os.path.basename(os.path.dirname(os.path.dirname(makefile)))
matches = [arch for arch in TauMagic.all() if arch.name == tau_arch_name]
if len(matches) == 1:
tau_arch = matches[0]
elif len(matches) == 0:
raise ConfigurationError("TAU Makefile '%s' targets an unrecognized TAU architecture: %s" %
(makefile, tau_arch_name))
else:
for arch in matches:
if arch.architecture == HOST_ARCH and arch.operating_system == HOST_OS:
tau_arch = arch
break
else:
parts = ["TAU Makefile '%s' targets an ambiguous TAU architecture: %s" % (makefile, tau_arch_name),
"It could be any of these:"]
parts.extend([" - %s on %s" % (arch.operating_system.name, arch.architecture.name)
for arch in matches])
raise ConfigurationError("\n".join(parts))
self.logger.info("Parsing TAU Makefile '%s' to populate command line arguments:", makefile)
args.host_arch = tau_arch.architecture.name
self.logger.info(" --host-arch='%s'", args.host_arch)
args.host_os = tau_arch.operating_system.name
self.logger.info(" --host-os='%s'", args.host_os)
args.tau_source = os.path.abspath(os.path.join(os.path.dirname(makefile), '..', '..'))
self.logger.info(" --taucmdr='%s'", args.tau_source)
with open(makefile, 'r') as fin:
compiler_parts = ("FULL_CC", "FULL_CXX", "TAU_F90")
package_parts = {"BFDINCLUDE": ("binutils_source", lambda x: os.path.dirname(x.lstrip("-I"))),
"UNWIND_INC": ("libunwind_source", lambda x: os.path.dirname(x.lstrip("-I"))),
"PAPIDIR": ("papi_source", os.path.abspath),
"PDTDIR": ("pdt_source", os.path.abspath),
"SCOREPDIR": ("scorep_source", os.path.abspath)}
tau_r = ''
for line in fin:
if line.startswith('#'):
continue
try:
key, val = [x.strip() for x in line.split('=', 1)]
except ValueError:
continue
if key == 'TAU_R':
tau_r = val.split()[0]
elif key in compiler_parts:
path = util.which(val.strip().split()[0].replace('$(TAU_R)', tau_r))
if not path:
self.logger.warning("Failed to parse %s in TAU Makefile '%s'", key, makefile)
continue
matching_info = Knowledgebase.find_compiler(os.path.basename(path))
if matching_info:
if len(matching_info) > 1:
self.logger.warning("Ambiguous compiler '%s' in TAU Makefile '%s'", path, makefile)
comp = InstalledCompiler(path, matching_info[0])
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path)
while comp.wrapped:
comp = comp.wrapped
attr = comp.info.role.keyword
setattr(args, attr, comp.absolute_path)
self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path)
elif key in package_parts:
attr, operator = package_parts[key]
path = val.strip()
if not path:
path = None
else:
path = operator(path)
if not os.path.exists(path):
self.logger.warning("'%s' referenced by TAU Makefile '%s' doesn't exist", path, makefile)
continue
setattr(args, attr, path)
self.logger.info(" --%s='%s'", attr.replace("_source", ""), path)
