def verify(self): """Check if the installation at :any:`installation_prefix` is valid. A valid installation provides all expected files and commands. Subclasses may wish to perform additional checks. Raises: SoftwarePackageError: Describs why the installation is invalid. """ LOGGER.debug("Verifying %s installation at '%s'", self.title, self.install_prefix) if not os.path.exists(self.install_prefix): raise SoftwarePackageError("'%s' does not exist" % self.install_prefix) for cmd in self.verify_commands: path = os.path.join(self.bin_path, cmd) if not os.path.exists(path): raise SoftwarePackageError("'%s' is missing" % path) if not os.access(path, os.X_OK): raise SoftwarePackageError("'%s' exists but is not executable" % path) for lib in self.verify_libraries: path = os.path.join(self.lib_path, lib) if not util.path_accessible(path): # Some systems (e.g. SuSE) append the machine bitwidth to the library path path = os.path.join(self.lib_path+'64', lib) if not util.path_accessible(path): raise SoftwarePackageError("'%s' is not accessible" % path) for header in self.verify_headers: path = os.path.join(self.include_path, header) if not util.path_accessible(path): raise SoftwarePackageError("'%s' is not accessible" % path) LOGGER.debug("%s installation at '%s' is valid", self.name, self.install_prefix)
def verify(self): """Check if the installation at :any:`installation_prefix` is valid. A valid installation provides all expected files and commands. Subclasses may wish to perform additional checks. Raises: SoftwarePackageError: Describs why the installation is invalid. """ LOGGER.debug("Verifying %s installation at '%s'", self.title, self.install_prefix) if not os.path.exists(self.install_prefix): raise SoftwarePackageError("'%s' does not exist" % self.install_prefix) for cmd in self.verify_commands: path = os.path.join(self.bin_path, cmd) if not os.path.exists(path): raise SoftwarePackageError("'%s' is missing" % path) if not os.access(path, os.X_OK): raise SoftwarePackageError( "'%s' exists but is not executable" % path) for lib in self.verify_libraries: path = os.path.join(self.lib_path, lib) if not util.path_accessible(path): # Some systems (e.g. SuSE) append the machine bitwidth to the library path path = os.path.join(self.lib_path + '64', lib) if not util.path_accessible(path): raise SoftwarePackageError("'%s' is not accessible" % path) for header in self.verify_headers: path = os.path.join(self.include_path, header) if not util.path_accessible(path): raise SoftwarePackageError("'%s' is not accessible" % path) LOGGER.debug("%s installation at '%s' is valid", self.name, self.install_prefix)
def __call__(self, parser, namespace, value, unused_option_string=None): """Sets the `self.dest` attribute in `namespace` to the parsed value of `value`. If `value` parses to a boolean True value then the attribute value is 'download'. If `value` parses to a boolean False value then the attribute value is ``None``. Otherwise the attribute value is the value of `value`. Args: parser (str): Argument parser object this group belongs to. namespace (object): Namespace to receive parsed value via setattr. value (str): Value parsed from the command line. """ try: value_as_bool = util.parse_bool( value, additional_true=['download', 'download-tr6', 'nightly']) except TypeError: if not util.is_url(value): value = os.path.abspath(os.path.expanduser(value)) if not (os.path.isdir(value) or util.path_accessible(value)): raise argparse.ArgumentError( self, "Keyword, valid path, or URL required: %s" % value) else: value = value.lower() if value_as_bool else None setattr(namespace, self.dest, value)
def _separate_launcher_cmd(cls, cmd): """Separate the launcher command and it's arguments from the application command(s) and arguments. Args: cmd (list): Command line. Returns: tuple: (Launcher command, Remainder of command line) Raises: ConfigurationError: No application config files or executables found after a recognized launcher command. """ # If '--' appears in the command then everything before it is a launcher + args # and everything after is the application + args try: idx = cmd.index('--') except ValueError: pass else: return cmd[:idx], cmd[idx + 1:] cmd0 = cmd[0] for launcher, appfile_flags in PROGRAM_LAUNCHERS.iteritems(): if launcher not in cmd0: continue # No '--' to indicate start of application, so look for first executable for idx, exe in enumerate(cmd[1:], 1): if util.which(exe): return cmd[:idx], cmd[idx:] # No exectuables, so look for application config file if appfile_flags: for i, arg in enumerate(cmd[1:], 1): try: arg, appfile = arg.split('=') except ValueError: try: appfile = cmd[i + 1] except IndexError: # Reached the end of the command line without finding an application config file break if arg in appfile_flags and util.path_accessible(appfile): return cmd, [] raise ConfigurationError( ("TAU is having trouble parsing the command line: no executable " "commands or %s application files were found after " "the launcher command '%s'") % (cmd0, cmd0), "Check that the command is correct. Does it work without TAU?", ("Use '--' to seperate '%s' and its arguments from the application " "command, e.g. `mpirun -np 4 -- ./a.out -l hello`" % cmd0)) # No launcher command, just an application command return [], cmd
def connect_database(self, *args, **kwargs): """Open the database for reading and writing.""" if self._database is None: util.mkdirp(self.prefix) dbfile = os.path.join(self.prefix, self.name + '.json') try: self._database = tinydb.TinyDB(dbfile, storage=_JsonFileStorage) except IOError as err: raise StorageError("Failed to access %s database '%s': %s" % (self.name, dbfile, err), "Check that you have `write` access") if not util.path_accessible(dbfile): raise StorageError("Database file '%s' exists but cannot be read." % dbfile, "Check that you have `read` access") LOGGER.debug("Initialized %s database '%s'", self.name, dbfile)
def _separate_launcher_cmd(cls, cmd): """Separate the launcher command and it's arguments from the application command(s) and arguments. Args: cmd (list): Command line. Returns: tuple: (Launcher command, Remainder of command line) Raises: ConfigurationError: No application config files or executables found after a recognized launcher command. """ # If '--' appears in the command then everything before it is a launcher + args # and everything after is the application + args try: idx = cmd.index('--') except ValueError: pass else: return cmd[:idx], cmd[idx+1:] cmd0 = cmd[0] for launcher, appfile_flags in PROGRAM_LAUNCHERS.iteritems(): if launcher not in cmd0: continue # No '--' to indicate start of application, so look for first executable for idx, exe in enumerate(cmd[1:], 1): if util.which(exe): return cmd[:idx], cmd[idx:] # No exectuables, so look for application config file if appfile_flags: for i, arg in enumerate(cmd[1:], 1): try: arg, appfile = arg.split('=') except ValueError: try: appfile = cmd[i+1] except IndexError: # Reached the end of the command line without finding an application config file break if arg in appfile_flags and util.path_accessible(appfile): return cmd, [] raise ConfigurationError(("TAU is having trouble parsing the command line: no executable " "commands or %s application files were found after " "the launcher command '%s'") % (cmd0, cmd0), "Check that the command is correct. Does it work without TAU?", ("Use '--' to seperate '%s' and its arguments from the application " "command, e.g. `mpirun -np 4 -- ./a.out -l hello`" % cmd0)) # No launcher command, just an application command return [], cmd
def connect_database(self, *args, **kwargs): """Open the database for reading and writing.""" if self._database is None: util.mkdirp(self.prefix) dbfile = os.path.join(self.prefix, self.name + '.json') try: storage = CachingMiddleware(_JsonFileStorage) storage.WRITE_CACHE_SIZE = 0 self._database = tinydb.TinyDB(dbfile, storage=storage) except IOError as err: raise StorageError("Failed to access %s database '%s': %s" % (self.name, dbfile, err), "Check that you have `write` access") if not util.path_accessible(dbfile): raise StorageError("Database file '%s' exists but cannot be read." % dbfile, "Check that you have `read` access") LOGGER.debug("Initialized %s database '%s'", self.name, dbfile)
def tau_source_default(): """"Per Sameer's request, override managed TAU installation with an existing unmanaged TAU installation. If a file named "override_tau_source" exists in the system-level storage prefix, use the contents of that file as the default path for TAU. Otherwise use "download" as the default. Returns: str: Path to TAU or "download". """ try: with open(os.path.join(SYSTEM_STORAGE.prefix, 'override_tau_source')) as fin: path = fin.read() except IOError: return 'download' path = path.strip() if not (os.path.isdir(path) and util.path_accessible(path)): LOGGER.warning("'%s' does not exist or is not accessable.") return 'download' return path
def __call__(self, parser, namespace, value, unused_option_string=None): """Sets the `self.dest` attribute in `namespace` to the parsed value of `value`. If `value` parses to a boolean True value then the attribute value is 'download'. If `value` parses to a boolean False value then the attribute value is ``None``. Otherwise the attribute value is the value of `value`. Args: parser (str): Argument parser object this group belongs to. namespace (object): Namespace to receive parsed value via setattr. value (str): Value parsed from the command line. """ try: value_as_bool = util.parse_bool(value, additional_true=['download', 'nightly']) except TypeError: if not util.is_url(value): value = os.path.abspath(os.path.expanduser(value)) if not (os.path.isdir(value) or util.path_accessible(value)): raise argparse.ArgumentError(self, "Keyword, valid path, or URL required: %s" % value) else: value = value.lower() if value_as_bool else None setattr(namespace, self.dest, value)
def _parse_tau_makefile(self, args): # Parsing a TAU Makefile is a really hairy operation, so let's lift the limits # pylint: disable=too-many-statements,too-many-locals makefile = args.forced_makefile if not util.path_accessible(makefile): self.parser.error("Invalid TAU makefile: %s" % makefile) tau_arch_name = os.path.basename( os.path.dirname(os.path.dirname(makefile))) matches = [ arch for arch in TauMagic.all() if arch.name == tau_arch_name ] if len(matches) == 1: tau_arch = matches[0] elif not matches: raise ConfigurationError( "TAU Makefile '%s' targets an unrecognized TAU architecture: %s" % (makefile, tau_arch_name)) else: for arch in matches: if arch.architecture == HOST_ARCH and arch.operating_system == HOST_OS: tau_arch = arch break else: parts = [ "TAU Makefile '%s' targets an ambiguous TAU architecture: %s" % (makefile, tau_arch_name), "It could be any of these:" ] parts.extend([ " - %s on %s" % (arch.operating_system.name, arch.architecture.name) for arch in matches ]) raise ConfigurationError("\n".join(parts)) self.logger.info( "Parsing TAU Makefile '%s' to populate command line arguments:", makefile) args.host_arch = tau_arch.architecture.name self.logger.info(" --host-arch='%s'", args.host_arch) args.host_os = tau_arch.operating_system.name self.logger.info(" --host-os='%s'", args.host_os) args.tau_source = os.path.abspath( os.path.join(os.path.dirname(makefile), '..', '..')) self.logger.info(" --taucmdr='%s'", args.tau_source) with open(makefile, 'r') as fin: compiler_parts = ("FULL_CC", "FULL_CXX", "TAU_F90") package_parts = { "BFDINCLUDE": ("binutils_source", lambda x: os.path.dirname(shlex.split(x)[0].lstrip("-I"))), "UNWIND_INC": ("libunwind_source", lambda x: os.path.dirname(x.lstrip("-I"))), "PAPIDIR": ("papi_source", os.path.abspath), "PDTDIR": ("pdt_source", os.path.abspath), "SCOREPDIR": ("scorep_source", os.path.abspath) } tau_r = '' for line in fin: if line.startswith('#'): continue try: key, val = [x.strip() for x in line.split('=', 1)] except ValueError: continue if key == 'TAU_R': tau_r = val.split()[0] elif key in compiler_parts: path = util.which(val.strip().split()[0].replace( '$(TAU_R)', tau_r)) if not path: self.logger.warning( "Failed to parse %s in TAU Makefile '%s'", key, makefile) continue matching_info = Knowledgebase.find_compiler( os.path.basename(path)) if matching_info: if len(matching_info) > 1: self.logger.warning( "Ambiguous compiler '%s' in TAU Makefile '%s'", path, makefile) comp = InstalledCompiler(path, matching_info[0]) attr = comp.info.role.keyword setattr(args, attr, comp.absolute_path) self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path) while comp.wrapped: comp = comp.wrapped attr = comp.info.role.keyword setattr(args, attr, comp.absolute_path) self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path) elif key in package_parts: attr, operator = package_parts[key] path = val.strip() if not path: path = None else: path = operator(path) if not os.path.exists(path): self.logger.warning( "'%s' referenced by TAU Makefile '%s' doesn't exist", path, makefile) continue setattr(args, attr, path) self.logger.info(" --%s='%s'", attr.replace("_source", ""), path)
def _parse_tau_makefile(self, args): # Parsing a TAU Makefile is a really hairy operation, so let's lift the limits # pylint: disable=too-many-statements,too-many-locals makefile = args.forced_makefile if not util.path_accessible(makefile): self.parser.error("Invalid TAU makefile: %s" % makefile) tau_arch_name = os.path.basename(os.path.dirname(os.path.dirname(makefile))) matches = [arch for arch in TauMagic.all() if arch.name == tau_arch_name] if len(matches) == 1: tau_arch = matches[0] elif len(matches) == 0: raise ConfigurationError("TAU Makefile '%s' targets an unrecognized TAU architecture: %s" % (makefile, tau_arch_name)) else: for arch in matches: if arch.architecture == HOST_ARCH and arch.operating_system == HOST_OS: tau_arch = arch break else: parts = ["TAU Makefile '%s' targets an ambiguous TAU architecture: %s" % (makefile, tau_arch_name), "It could be any of these:"] parts.extend([" - %s on %s" % (arch.operating_system.name, arch.architecture.name) for arch in matches]) raise ConfigurationError("\n".join(parts)) self.logger.info("Parsing TAU Makefile '%s' to populate command line arguments:", makefile) args.host_arch = tau_arch.architecture.name self.logger.info(" --host-arch='%s'", args.host_arch) args.host_os = tau_arch.operating_system.name self.logger.info(" --host-os='%s'", args.host_os) args.tau_source = os.path.abspath(os.path.join(os.path.dirname(makefile), '..', '..')) self.logger.info(" --taucmdr='%s'", args.tau_source) with open(makefile, 'r') as fin: compiler_parts = ("FULL_CC", "FULL_CXX", "TAU_F90") package_parts = {"BFDINCLUDE": ("binutils_source", lambda x: os.path.dirname(x.lstrip("-I"))), "UNWIND_INC": ("libunwind_source", lambda x: os.path.dirname(x.lstrip("-I"))), "PAPIDIR": ("papi_source", os.path.abspath), "PDTDIR": ("pdt_source", os.path.abspath), "SCOREPDIR": ("scorep_source", os.path.abspath)} tau_r = '' for line in fin: if line.startswith('#'): continue try: key, val = [x.strip() for x in line.split('=', 1)] except ValueError: continue if key == 'TAU_R': tau_r = val.split()[0] elif key in compiler_parts: path = util.which(val.strip().split()[0].replace('$(TAU_R)', tau_r)) if not path: self.logger.warning("Failed to parse %s in TAU Makefile '%s'", key, makefile) continue matching_info = Knowledgebase.find_compiler(os.path.basename(path)) if matching_info: if len(matching_info) > 1: self.logger.warning("Ambiguous compiler '%s' in TAU Makefile '%s'", path, makefile) comp = InstalledCompiler(path, matching_info[0]) attr = comp.info.role.keyword setattr(args, attr, comp.absolute_path) self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path) while comp.wrapped: comp = comp.wrapped attr = comp.info.role.keyword setattr(args, attr, comp.absolute_path) self.logger.info(" --%s='%s'", attr.lower().replace("_", "-"), comp.absolute_path) elif key in package_parts: attr, operator = package_parts[key] path = val.strip() if not path: path = None else: path = operator(path) if not os.path.exists(path): self.logger.warning("'%s' referenced by TAU Makefile '%s' doesn't exist", path, makefile) continue setattr(args, attr, path) self.logger.info(" --%s='%s'", attr.replace("_source", ""), path)