def _get_conda_package_details(self, conda_path): packages = {} file_to_package_map = {} for meta_file in self._get_conda_meta_files(conda_path): try: out, err = self._session.execute_command('cat %s' % meta_file) details = json.loads(out) # print meta_file # print(json.dumps(details, indent=4)) if "name" in details: lgr.debug("Found conda package %s", details["name"]) # Packages are recorded in the conda environment as # name=version=build conda_package_name = \ ("%s=%s=%s" % (details["name"], details["version"], details["build"])) packages[conda_package_name] = details # Now map the package files to the package for f in details["files"]: full_path = os.path.normpath( os.path.join(conda_path, f)) file_to_package_map[full_path] = conda_package_name except Exception as exc: lgr.warning("Could not retrieve conda info in path %s: %s", conda_path, exc_str(exc)) return packages, file_to_package_map
def _venv_exe_path(self): try: out, _ = self._session.execute_command(["which", "virtualenv"]) except Exception as exc: lgr.debug("Could not determine virtualenv path: %s", exc_str(exc)) return return out.strip()
def _get_conda_path(self, path): paths = [] conda_path = None while path not in {None, os.path.pathsep, '', '/'}: if path in self._paths_cache: conda_path = self._paths_cache[path] break paths.append(path) try: _, _ = self._session.execute_command( 'ls -ld %s/bin/conda %s/conda-meta' % (path, path) ) except Exception as exc: lgr.debug("Did not detect conda at the path %s: %s", path, exc_str(exc)) path = os.path.dirname(path) # go to the parent continue conda_path = path lgr.info("Detected conda %s", conda_path) break for path in paths: self._paths_cache[path] = conda_path return conda_path
def get_at_dirpath(cls, session, dirpath): # ho ho -- no longer the case that there is .svn in each subfolder: # http://stackoverflow.com/a/9070242 found = False if exists(opj(dirpath, '.svn')): # early detection found = True # but still might be under SVN if not found: try: out, err = session.execute_command( 'svn info', # expect_fail=True, cwd=dirpath) except CommandError as exc: if "Please see the 'svn upgrade' command" in str(exc): lgr.warning("SVN at %s is outdated, needs 'svn upgrade'", dirpath) else: # we are in SVN but it is outdated and needs an upgrade lgr.debug("Probably %s is not under SVN repo path: %s", dirpath, exc_str(exc)) return None # for now we treat each directory under SVN independently # pros: # - could provide us 'minimal' set of checkouts to do, since it might be # quite expensive to checkout the entire tree if it was not used # besides few leaves lgr.debug("Detected SVN repository at %s", dirpath) return cls(dirpath, session=session)
def _venv_version(self): try: out, _ = self._session.execute_command(["virtualenv", "--version"]) except Exception as exc: lgr.debug("Could not determine virtualenv version: %s", exc_str(exc)) return return out.strip()
def _get_conda_meta_files(self, conda_path): try: out, _ = self._session.execute_command('ls %s/conda-meta/*.json' % conda_path) return iter(out.splitlines()) except Exception as exc: # Empty conda environment (unusual situation) lgr.warning("Could not retrieve conda-meta files in path %s: %s", conda_path, exc_str(exc)) return iter(())
def _is_venv_directory(self, path): try: self._session.execute_command(["grep", "-q", "VIRTUAL_ENV", "{}/bin/activate".format(path)]) except Exception as exc: lgr.debug("Did not detect virtualenv at the path %s: %s", path, exc_str(exc)) return False return True
def _get_conda_info(self, conda_path): details = {} try: out, err = self._session.execute_command( '%s/bin/conda info --json' % conda_path) details = json.loads(out) except Exception as exc: lgr.warning("Could not retrieve conda info in path %s: %s", conda_path, exc_str(exc)) return details
def _get_conda_env_export(self, root_prefix, conda_path): export = {} try: # NOTE: We need to call conda-env directly. Conda has problems # calling conda-env without a PATH being set... out, err = self._session.execute_command( '%s/bin/conda-env export -p %s' % (root_prefix, conda_path)) export = yaml.load(out) except Exception as exc: if "unrecognized arguments: -p" in exc_str(exc): lgr.warning( "Could not retrieve conda environment " "export from path %s: " "Please use Conda 4.3.19 or greater", conda_path) else: lgr.warning( "Could not retrieve conda environment " "export from path %s: %s", conda_path, exc_str(exc)) return export
def __getitem__(self, module): # when ran straight in its source code -- fails to discover nipy's version.. TODO #if module == 'nipy': # import pdb; pdb.set_trace() if not isinstance(module, string_types): modname = module.__name__ else: modname = module module = None # Early returns None so we do not store prev result for them # and allow users to install things at run time, so later check # doesn't pick it up from the _versions if modname not in self._versions: version = None # by default -- not present if modname in self.CUSTOM: try: version = self.CUSTOM[modname]() version = self._deduce_version(version) except Exception as exc: lgr.debug("Failed to deduce version of %s due to %s" % (modname, exc_str(exc))) return None else: if module is None: if modname not in sys.modules: try: module = __import__(modname) except ImportError: lgr.debug("Module %s seems to be not present" % modname) return None except Exception as exc: lgr.warning("Failed to import module %s due to %s", modname, exc_str(exc)) return None else: module = sys.modules[modname] if module: version = self._deduce_version(module) self._versions[modname] = version return self._versions.get(modname, self.UNKNOWN)
def _get_package_details(self, venv_path): pip = venv_path + "/bin/pip" try: packages, file_to_pkg = piputils.get_package_details( self._session, pip) except Exception as exc: lgr.warning("Could not determine pip package details for %s: %s", venv_path, exc_str(exc)) return {}, {} return packages, file_to_pkg
def _python_version(self, venv_path): try: out, err = self._session.execute_command( [venv_path + "/bin/python", "--version"]) # Python 2 sends its version to stderr, while Python 3 # sends it to stdout. Version has format "Python pyver = out if "Python" in out else err return pyver.strip().split()[1] except Exception as exc: lgr.debug("Could not determine python version: %s", exc_str(exc)) return
def isdir(self, path): try: out, err = self.execute_command(self.isdir_command(path)) except Exception as exc: # TODO: More specific exception? lgr.debug("Check for directory failed: %s", exc_str(exc)) return False if out == 'Found\n': return True else: lgr.debug( "Standard error was not empty (%r), thus assuming that " "test for direcory has failed", err) return False
def exists(self, path): """Return if file exists""" try: out, err = self.execute_command(["[", "-e", path, "]"]) except Exception as exc: # TODO: More specific exception? lgr.debug("Check for file presence failed: %s", exc_str(exc)) return False if not err: return True else: lgr.debug( "Standard error was not empty (%r), thus assuming that " "test for file presence has failed", err)
def isdir(self, path): try: command = ['test', '-d', path, '&&', 'echo', 'Found'] out, err = self.execute_command(['bash', '-c', ' '.join(command)]) except Exception as exc: # TODO: More specific exception? lgr.debug("Check for directory failed: %s", exc_str(exc)) return False if out == 'Found\n': return True else: lgr.debug( "Standard error was not empty (%r), thus assuming that " "test for direcory has failed", err) return False
def get_at_dirpath(cls, session, dirpath): try: out, err = session.execute_command( 'git rev-parse --show-toplevel', # expect_fail=True, cwd=dirpath) except CommandError as exc: lgr.debug("Probably %s is not under git repo path: %s", dirpath, exc_str(exc)) return None topdir = out.rstrip('\n') lgr.debug( "Detected Git repository at %s for %s. Creating a session shim", topdir, dirpath) return cls(topdir, session=session)
def _get_system_ssh_version(): """Return version of ssh available system-wide """ try: out, err = _runner.run('ssh -V'.split(), expect_fail=True, expect_stderr=True) # apparently spits out to err but I wouldn't trust it blindly if err.startswith('OpenSSH'): out = err assert out.startswith( 'OpenSSH') # that is the only one we care about atm return out.split(' ', 1)[0].rstrip(',.').split('_')[1] except CommandError as exc: lgr.debug("Could not determine version of ssh available: %s", exc_str(exc)) return None
def get_resource_classes(names=None): for name in names or ResourceManager._discover_types(): try: module = import_module('niceman.resource.{}'.format(name)) except ImportError as exc: import difflib known = ResourceManager._discover_types() suggestions = difflib.get_close_matches(name, known) lgr.warning( "Failed to import resource %s: %s. %s: %s", name, exc_str(exc), "Similar backends" if suggestions else "Known backends", ', '.join(suggestions or known)) continue class_name = ''.join([token.capitalize() for token in name.split('_')]) cls = getattr(module, class_name) if issubclass(cls, Resource): yield name, cls else: lgr.debug( "Skipping %s.%s because it is not a Resource. " "Consider moving away", module, class_name)
def _get_conda_pip_package_details(self, env_export, conda_path): packages = {} file_to_package_map = {} dependencies = env_export.get("dependencies") pip_deps = [] for dep in dependencies: if isinstance(dep, dict) and "pip" in dep: pip_deps = dep.get("pip") for pip_dep in pip_deps: name, origin_location = self.parse_pip_package_entry(pip_dep) try: out, err = self._session.execute_command( '%s/bin/pip show -f %s' % (conda_path, name) ) pip_info = self._parse_pip_show(out) # Record the details we care about details = {"name": pip_info.get("Name"), "version": pip_info.get("Version"), "installer": "pip", "origin_location": origin_location} packages[pip_dep] = details # Map the package files to the package for f in pip_info.get("Files"): full_path = os.path.normpath( os.path.join(pip_info.get("Location"), f)) file_to_package_map[full_path] = pip_dep except Exception as exc: lgr.warning("Could not retrieve pip info " "export from path %s: %s", conda_path, exc_str(exc)) continue return packages, file_to_package_map