def _venv_exe_path(self):
try:
out, _ = self._session.execute_command(["which", "virtualenv"])
except Exception as exc:
lgr.debug("Could not determine virtualenv path: %s", exc_str(exc))
return
return out.strip()
def _get_conda_package_details(self, conda_path):
packages = {}
file_to_package_map = {}
for meta_file in self._get_conda_meta_files(conda_path):
try:
out, err = self._session.execute_command('cat %s' % meta_file)
details = json.loads(out)
# print meta_file
# print(json.dumps(details, indent=4))
if "name" in details:
lgr.debug("Found conda package %s", details["name"])
# Packages are recorded in the conda environment as
# name=version=build
conda_package_name = \
("%s=%s=%s" % (details["name"], details["version"],
details["build"]))
packages[conda_package_name] = details
# Now map the package files to the package
for f in details["files"]:
full_path = os.path.normpath(
os.path.join(conda_path, f))
file_to_package_map[full_path] = conda_package_name
except Exception as exc:
lgr.warning("Could not retrieve conda info in path %s: %s",
conda_path, exc_str(exc))
return packages, file_to_package_map
def get_at_dirpath(cls, session, dirpath):
# ho ho -- no longer the case that there is .svn in each subfolder:
# http://stackoverflow.com/a/9070242
found = False
if exists(opj(dirpath, '.svn')): # early detection
found = True
# but still might be under SVN
if not found:
try:
out, err = session.execute_command(
'svn info',
# expect_fail=True,
cwd=dirpath)
except CommandError as exc:
if "Please see the 'svn upgrade' command" in str(exc):
lgr.warning("SVN at %s is outdated, needs 'svn upgrade'",
dirpath)
else:
# we are in SVN but it is outdated and needs an upgrade
lgr.debug("Probably %s is not under SVN repo path: %s",
dirpath, exc_str(exc))
return None
# for now we treat each directory under SVN independently
# pros:
# - could provide us 'minimal' set of checkouts to do, since it might be
# quite expensive to checkout the entire tree if it was not used
# besides few leaves
lgr.debug("Detected SVN repository at %s", dirpath)
return cls(dirpath, session=session)
def _venv_version(self):
try:
out, _ = self._session.execute_command(["virtualenv", "--version"])
except Exception as exc:
lgr.debug("Could not determine virtualenv version: %s",
exc_str(exc))
return
return out.strip()
def fetch(self, on_remote_finish=None):
"""Fetch the results from the remote dataset sibling.
Parameters
----------
on_remote_finish : callable, optional
Function to be called when work with the resource is finished. It
will be passed two arguments, the resource and the failed subjobs
(list of ints).
"""
from datalad.support.exceptions import CommandError as DCError
lgr.info("Fetching results for %s", self.jobid)
failed = self.get_failed_subjobs()
resource_name = self.resource.name
ref = self.job_refname
lgr.info("Updating local dataset with changes from '%s'",
resource_name)
repo = self.ds.repo
repo.fetch(resource_name,
"{0}:{0}".format(ref),
recurse_submodules="no")
failure = False
try:
repo.merge(ref)
except DCError as exc:
lgr.warning(
"Failed to merge in changes from %s. "
"Check %s for merge conflicts. %s", ref, self.ds.path,
exc_str(exc))
failure = True
else:
# Handle any subdataset updates. We could avoid this if we knew
# there were no subdataset changes, but it's probably simplest to
# just unconditionally call update().
failure = False
try:
call_check_dl_results(self.ds.update,
"'datalad update' failed",
sibling=resource_name,
merge=True,
follow="parentds",
recursive=True,
on_failure="ignore")
except OrchestratorError:
failure = True
if not failure:
lgr.info("Getting outputs from '%s'", resource_name)
outputs = list(self.get_outputs())
if outputs:
self.ds.get(path=outputs)
self.log_failed(failed, func=lambda mdir, _: self.ds.get(path=mdir))
lgr.info("Finished with remote resource '%s'", resource_name)
if on_remote_finish:
on_remote_finish(self.resource, failed)
def _is_venv_directory(self, path):
try:
self._session.execute_command(
["grep", "-q", "VIRTUAL_ENV", "{}/bin/activate".format(path)])
except Exception as exc:
lgr.debug("Did not detect virtualenv at the path %s: %s", path,
exc_str(exc))
return False
return True
def _get_conda_meta_files(self, conda_path):
try:
out, _ = self._session.execute_command('ls %s/conda-meta/*.json' %
conda_path)
return iter(out.splitlines())
except Exception as exc: # Empty conda environment (unusual situation)
lgr.warning("Could not retrieve conda-meta files in path %s: %s",
conda_path, exc_str(exc))
return iter(())
def _get_package_details(self, venv_path):
pip = venv_path + "/bin/pip"
try:
packages, file_to_pkg = piputils.get_package_details(
self._session, pip)
except Exception as exc:
lgr.warning("Could not determine pip package details for %s: %s",
venv_path, exc_str(exc))
return {}, {}
return packages, file_to_pkg
def __getitem__(self, module):
# when ran straight in its source code -- fails to discover nipy's version.. TODO
#if module == 'nipy':
# import pdb; pdb.set_trace()
if not isinstance(module, str):
modname = module.__name__
else:
modname = module
module = None
# Early returns None so we do not store prev result for them
# and allow users to install things at run time, so later check
# doesn't pick it up from the _versions
if modname not in self._versions:
version = None # by default -- not present
if modname in self.CUSTOM:
try:
version = self.CUSTOM[modname]()
version = self._deduce_version(version)
except Exception as exc:
lgr.debug("Failed to deduce version of %s due to %s" %
(modname, exc_str(exc)))
return None
else:
if module is None:
if modname not in sys.modules:
try:
module = __import__(modname)
except ImportError:
lgr.debug("Module %s seems to be not present" %
modname)
return None
except Exception as exc:
lgr.warning("Failed to import module %s due to %s",
modname, exc_str(exc))
return None
else:
module = sys.modules[modname]
if module:
version = self._deduce_version(module)
self._versions[modname] = version
return self._versions.get(modname, self.UNKNOWN)
def _get_conda_info(self, conda_path):
details = {}
try:
out, err = self._session.execute_command(
'%s/bin/conda info --json' % conda_path)
details = json.loads(out)
except Exception as exc:
lgr.warning("Could not retrieve conda info in path %s: %s",
conda_path, exc_str(exc))
return details
def _get_conda_env_export(self, root_prefix, conda_path):
export = {}
try:
# NOTE: We need to call conda-env directly. Conda has problems
# calling conda-env without a PATH being set...
out, err = self._session.execute_command(
'%s/bin/conda-env export -p %s' % (root_prefix, conda_path))
export = yaml.safe_load(out)
except Exception as exc:
if "unrecognized arguments: -p" in exc_str(exc):
lgr.warning(
"Could not retrieve conda environment "
"export from path %s: "
"Please use Conda 4.3.19 or greater", conda_path)
else:
lgr.warning(
"Could not retrieve conda environment "
"export from path %s: %s", conda_path, exc_str(exc))
return export
def _python_version(self, venv_path):
try:
out, err = self._session.execute_command(
[venv_path + "/bin/python", "--version"])
# Python 2 sends its version to stderr, while Python 3
# sends it to stdout.
pyver = out if "Python" in out else err
return pyver.strip().split()[1]
except Exception as exc:
lgr.debug("Could not determine python version: %s", exc_str(exc))
return
def isdir(self, path):
try:
out, err = self.execute_command(self.isdir_command(path))
except Exception as exc: # TODO: More specific exception?
lgr.debug("Check for directory failed: %s", exc_str(exc))
return False
if out == 'Found\n':
return True
else:
lgr.debug("Standard error was not empty (%r), thus assuming that "
"test for direcory has failed", err)
return False
def exists(self, path):
"""Return if file exists"""
try:
out, err = self.execute_command(self.exists_command(path))
except Exception as exc: # TODO: More specific exception?
lgr.debug("Check for file presence failed: %s", exc_str(exc))
return False
if out == 'Found\n':
return True
else:
lgr.debug("Standard error was not empty (%r), thus assuming that "
"test for file presence has failed", err)
return False
def get_resource_classes(names=None):
for name in names or discover_types():
try:
cls = get_resource_class(name)
except ResourceError as exc:
lgr.warning(exc_str(exc))
continue
if issubclass(cls, Resource):
yield name, cls
else:
lgr.debug(
"Skipping %s because it is not a Resource. "
"Consider moving away", cls)
def _get_system_ssh_version():
"""Return version of ssh available system-wide
"""
try:
out, err = _runner.run('ssh -V'.split(),
expect_fail=True, expect_stderr=True)
# apparently spits out to err but I wouldn't trust it blindly
if err.startswith('OpenSSH'):
out = err
assert out.startswith('OpenSSH') # that is the only one we care about atm
return out.split(' ', 1)[0].rstrip(',.').split('_')[1]
except CommandError as exc:
lgr.debug("Could not determine version of ssh available: %s", exc_str(exc))
return None
def get_at_dirpath(cls, session, dirpath):
try:
out, err = session.execute_command(
'git rev-parse --show-toplevel',
# expect_fail=True,
cwd=dirpath)
except CommandError as exc:
lgr.debug("Probably %s is not under git repo path: %s", dirpath,
exc_str(exc))
return None
topdir = out.rstrip('\n')
lgr.debug(
"Detected Git repository at %s for %s. Creating a session shim",
topdir, dirpath)
return cls(topdir, session=session)
def __call__(queries, action="auto", all_=False, status=False):
job_files = LREG.find_job_files()
if not job_files:
lgr.info("No jobs found")
return
if all_:
matched_ids = job_files.keys()
else:
matched_ids = []
for query in queries:
m = match(query, job_files)
if m:
matched_ids.append(m)
else:
lgr.warning("No jobs matched query %s", query)
if not matched_ids and action in ["delete", "fetch"]:
# These are actions where we don't want to just conveniently
# default to "all" unless --all is explicitly specified.
raise ValueError("Must specify jobs to {}".format(action))
# We don't need to load the job to delete it, so check that first.
if action == "delete":
for i in matched_ids:
LREG.unregister(i)
else:
jobs = [_load(job_files[i]) for i in matched_ids or job_files]
if action == "fetch" or (action == "auto" and matched_ids):
fn = fetch
elif action == "list" or action == "auto":
fn = partial(show_oneline, status=status)
elif action == "show":
fn = partial(show, status=status)
else:
raise RuntimeError("Unknown action: {}".format(action))
for job in jobs:
try:
fn(job)
except OrchestratorError as exc:
lgr.error("job %s failed: %s", job["_jobid"], exc_str(exc))
except ResourceNotFoundError:
lgr.error("Resource %s (%s) no longer exists",
job["resource_id"], job["resource_name"])
def status(self):
"""Like Orchestrator.status, but inspect the job's git ref if needed.
"""
status = super(DataladOrchestrator, self).status
if status == "unknown":
# The local tree might be different because of another just. Check
# the ref for the status.
try:
status_from_ref = self._execute_in_wdir(
"git cat-file -p {}:{}"
.format(self.job_refname,
# FIXME: How to handle subjobs?
op.relpath(op.join(self.meta_directory, "status.0"),
self.working_directory)))
except OrchestratorError as exc:
# Most likely the ref was never created because the runscript
# failed. Let follow() signal the error.
lgr.debug("Failed to get status from %s tree: %s",
self.job_refname, exc_str(exc))
else:
status = status_from_ref.strip() or status
return status
def get_resource_classes(names=None):
for name in names or ResourceManager._discover_types():
try:
module = import_module('reproman.resource.{}'.format(name))
except ImportError as exc:
import difflib
known = ResourceManager._discover_types()
suggestions = difflib.get_close_matches(name, known)
lgr.warning(
"Failed to import resource %s: %s. %s: %s", name, exc_str(exc),
"Similar backends" if suggestions else "Known backends",
', '.join(suggestions or known))
continue
class_name = ''.join([token.capitalize() for token in name.split('_')])
cls = getattr(module, class_name)
if issubclass(cls, Resource):
yield name, cls
else:
lgr.debug(
"Skipping %s.%s because it is not a Resource. "
"Consider moving away", module, class_name)
