def _get_conda_package_details(self, conda_path):
packages = {}
file_to_package_map = {}
for meta_file in self._get_conda_meta_files(conda_path):
try:
out, err = self._session.execute_command('cat %s' % meta_file)
details = json.loads(out)
# print meta_file
# print(json.dumps(details, indent=4))
if "name" in details:
lgr.debug("Found conda package %s", details["name"])
# Packages are recorded in the conda environment as
# name=version=build
conda_package_name = \
("%s=%s=%s" % (details["name"], details["version"],
details["build"]))
packages[conda_package_name] = details
# Now map the package files to the package
for f in details["files"]:
full_path = os.path.normpath(
os.path.join(conda_path, f))
file_to_package_map[full_path] = conda_package_name
except Exception as exc:
lgr.warning("Could not retrieve conda info in path %s: %s",
conda_path, exc_str(exc))
return packages, file_to_package_map
def _venv_exe_path(self):
try:
out, _ = self._session.execute_command(["which", "virtualenv"])
except Exception as exc:
lgr.debug("Could not determine virtualenv path: %s", exc_str(exc))
return
return out.strip()
def _get_conda_path(self, path):
paths = []
conda_path = None
while path not in {None, os.path.pathsep, '', '/'}:
if path in self._paths_cache:
conda_path = self._paths_cache[path]
break
paths.append(path)
try:
_, _ = self._session.execute_command(
'ls -ld %s/bin/conda %s/conda-meta'
% (path, path)
)
except Exception as exc:
lgr.debug("Did not detect conda at the path %s: %s", path,
exc_str(exc))
path = os.path.dirname(path) # go to the parent
continue
conda_path = path
lgr.info("Detected conda %s", conda_path)
break
for path in paths:
self._paths_cache[path] = conda_path
return conda_path
def get_at_dirpath(cls, session, dirpath):
# ho ho -- no longer the case that there is .svn in each subfolder:
# http://stackoverflow.com/a/9070242
found = False
if exists(opj(dirpath, '.svn')): # early detection
found = True
# but still might be under SVN
if not found:
try:
out, err = session.execute_command(
'svn info',
# expect_fail=True,
cwd=dirpath)
except CommandError as exc:
if "Please see the 'svn upgrade' command" in str(exc):
lgr.warning("SVN at %s is outdated, needs 'svn upgrade'",
dirpath)
else:
# we are in SVN but it is outdated and needs an upgrade
lgr.debug("Probably %s is not under SVN repo path: %s",
dirpath, exc_str(exc))
return None
# for now we treat each directory under SVN independently
# pros:
# - could provide us 'minimal' set of checkouts to do, since it might be
# quite expensive to checkout the entire tree if it was not used
# besides few leaves
lgr.debug("Detected SVN repository at %s", dirpath)
return cls(dirpath, session=session)
def _venv_version(self):
try:
out, _ = self._session.execute_command(["virtualenv", "--version"])
except Exception as exc:
lgr.debug("Could not determine virtualenv version: %s",
exc_str(exc))
return
return out.strip()
def _get_conda_meta_files(self, conda_path):
try:
out, _ = self._session.execute_command('ls %s/conda-meta/*.json' %
conda_path)
return iter(out.splitlines())
except Exception as exc: # Empty conda environment (unusual situation)
lgr.warning("Could not retrieve conda-meta files in path %s: %s",
conda_path, exc_str(exc))
return iter(())
def _is_venv_directory(self, path):
try:
self._session.execute_command(["grep", "-q", "VIRTUAL_ENV",
"{}/bin/activate".format(path)])
except Exception as exc:
lgr.debug("Did not detect virtualenv at the path %s: %s",
path, exc_str(exc))
return False
return True
def _get_conda_info(self, conda_path):
details = {}
try:
out, err = self._session.execute_command(
'%s/bin/conda info --json' % conda_path)
details = json.loads(out)
except Exception as exc:
lgr.warning("Could not retrieve conda info in path %s: %s",
conda_path, exc_str(exc))
return details
def _get_conda_env_export(self, root_prefix, conda_path):
export = {}
try:
# NOTE: We need to call conda-env directly. Conda has problems
# calling conda-env without a PATH being set...
out, err = self._session.execute_command(
'%s/bin/conda-env export -p %s' % (root_prefix, conda_path))
export = yaml.load(out)
except Exception as exc:
if "unrecognized arguments: -p" in exc_str(exc):
lgr.warning(
"Could not retrieve conda environment "
"export from path %s: "
"Please use Conda 4.3.19 or greater", conda_path)
else:
lgr.warning(
"Could not retrieve conda environment "
"export from path %s: %s", conda_path, exc_str(exc))
return export
def __getitem__(self, module):
# when ran straight in its source code -- fails to discover nipy's version.. TODO
#if module == 'nipy':
# import pdb; pdb.set_trace()
if not isinstance(module, string_types):
modname = module.__name__
else:
modname = module
module = None
# Early returns None so we do not store prev result for them
# and allow users to install things at run time, so later check
# doesn't pick it up from the _versions
if modname not in self._versions:
version = None # by default -- not present
if modname in self.CUSTOM:
try:
version = self.CUSTOM[modname]()
version = self._deduce_version(version)
except Exception as exc:
lgr.debug("Failed to deduce version of %s due to %s" %
(modname, exc_str(exc)))
return None
else:
if module is None:
if modname not in sys.modules:
try:
module = __import__(modname)
except ImportError:
lgr.debug("Module %s seems to be not present" %
modname)
return None
except Exception as exc:
lgr.warning("Failed to import module %s due to %s",
modname, exc_str(exc))
return None
else:
module = sys.modules[modname]
if module:
version = self._deduce_version(module)
self._versions[modname] = version
return self._versions.get(modname, self.UNKNOWN)
def _get_package_details(self, venv_path):
pip = venv_path + "/bin/pip"
try:
packages, file_to_pkg = piputils.get_package_details(
self._session, pip)
except Exception as exc:
lgr.warning("Could not determine pip package details for %s: %s",
venv_path, exc_str(exc))
return {}, {}
return packages, file_to_pkg
def _python_version(self, venv_path):
try:
out, err = self._session.execute_command(
[venv_path + "/bin/python", "--version"])
# Python 2 sends its version to stderr, while Python 3
# sends it to stdout. Version has format "Python
pyver = out if "Python" in out else err
return pyver.strip().split()[1]
except Exception as exc:
lgr.debug("Could not determine python version: %s", exc_str(exc))
return
def isdir(self, path):
try:
out, err = self.execute_command(self.isdir_command(path))
except Exception as exc: # TODO: More specific exception?
lgr.debug("Check for directory failed: %s", exc_str(exc))
return False
if out == 'Found\n':
return True
else:
lgr.debug(
"Standard error was not empty (%r), thus assuming that "
"test for direcory has failed", err)
return False
def exists(self, path):
"""Return if file exists"""
try:
out, err = self.execute_command(["[", "-e", path, "]"])
except Exception as exc: # TODO: More specific exception?
lgr.debug("Check for file presence failed: %s", exc_str(exc))
return False
if not err:
return True
else:
lgr.debug(
"Standard error was not empty (%r), thus assuming that "
"test for file presence has failed", err)
def isdir(self, path):
try:
command = ['test', '-d', path, '&&', 'echo', 'Found']
out, err = self.execute_command(['bash', '-c', ' '.join(command)])
except Exception as exc: # TODO: More specific exception?
lgr.debug("Check for directory failed: %s", exc_str(exc))
return False
if out == 'Found\n':
return True
else:
lgr.debug(
"Standard error was not empty (%r), thus assuming that "
"test for direcory has failed", err)
return False
def get_at_dirpath(cls, session, dirpath):
try:
out, err = session.execute_command(
'git rev-parse --show-toplevel',
# expect_fail=True,
cwd=dirpath)
except CommandError as exc:
lgr.debug("Probably %s is not under git repo path: %s", dirpath,
exc_str(exc))
return None
topdir = out.rstrip('\n')
lgr.debug(
"Detected Git repository at %s for %s. Creating a session shim",
topdir, dirpath)
return cls(topdir, session=session)
def _get_system_ssh_version():
"""Return version of ssh available system-wide
"""
try:
out, err = _runner.run('ssh -V'.split(),
expect_fail=True,
expect_stderr=True)
# apparently spits out to err but I wouldn't trust it blindly
if err.startswith('OpenSSH'):
out = err
assert out.startswith(
'OpenSSH') # that is the only one we care about atm
return out.split(' ', 1)[0].rstrip(',.').split('_')[1]
except CommandError as exc:
lgr.debug("Could not determine version of ssh available: %s",
exc_str(exc))
return None
def get_resource_classes(names=None):
for name in names or ResourceManager._discover_types():
try:
module = import_module('niceman.resource.{}'.format(name))
except ImportError as exc:
import difflib
known = ResourceManager._discover_types()
suggestions = difflib.get_close_matches(name, known)
lgr.warning(
"Failed to import resource %s: %s. %s: %s", name, exc_str(exc),
"Similar backends" if suggestions else "Known backends",
', '.join(suggestions or known))
continue
class_name = ''.join([token.capitalize() for token in name.split('_')])
cls = getattr(module, class_name)
if issubclass(cls, Resource):
yield name, cls
else:
lgr.debug(
"Skipping %s.%s because it is not a Resource. "
"Consider moving away", module, class_name)
def _get_conda_pip_package_details(self, env_export, conda_path):
packages = {}
file_to_package_map = {}
dependencies = env_export.get("dependencies")
pip_deps = []
for dep in dependencies:
if isinstance(dep, dict) and "pip" in dep:
pip_deps = dep.get("pip")
for pip_dep in pip_deps:
name, origin_location = self.parse_pip_package_entry(pip_dep)
try:
out, err = self._session.execute_command(
'%s/bin/pip show -f %s'
% (conda_path, name)
)
pip_info = self._parse_pip_show(out)
# Record the details we care about
details = {"name": pip_info.get("Name"),
"version": pip_info.get("Version"),
"installer": "pip",
"origin_location": origin_location}
packages[pip_dep] = details
# Map the package files to the package
for f in pip_info.get("Files"):
full_path = os.path.normpath(
os.path.join(pip_info.get("Location"), f))
file_to_package_map[full_path] = pip_dep
except Exception as exc:
lgr.warning("Could not retrieve pip info "
"export from path %s: %s", conda_path,
exc_str(exc))
continue
return packages, file_to_package_map
