def _venv_exe_path(self): try: out, _ = self._session.execute_command(["which", "virtualenv"]) except Exception as exc: lgr.debug("Could not determine virtualenv path: %s", exc_str(exc)) return return out.strip()
def _get_conda_package_details(self, conda_path): packages = {} file_to_package_map = {} for meta_file in self._get_conda_meta_files(conda_path): try: out, err = self._session.execute_command('cat %s' % meta_file) details = json.loads(out) # print meta_file # print(json.dumps(details, indent=4)) if "name" in details: lgr.debug("Found conda package %s", details["name"]) # Packages are recorded in the conda environment as # name=version=build conda_package_name = \ ("%s=%s=%s" % (details["name"], details["version"], details["build"])) packages[conda_package_name] = details # Now map the package files to the package for f in details["files"]: full_path = os.path.normpath( os.path.join(conda_path, f)) file_to_package_map[full_path] = conda_package_name except Exception as exc: lgr.warning("Could not retrieve conda info in path %s: %s", conda_path, exc_str(exc)) return packages, file_to_package_map
def get_at_dirpath(cls, session, dirpath): # ho ho -- no longer the case that there is .svn in each subfolder: # http://stackoverflow.com/a/9070242 found = False if exists(opj(dirpath, '.svn')): # early detection found = True # but still might be under SVN if not found: try: out, err = session.execute_command( 'svn info', # expect_fail=True, cwd=dirpath) except CommandError as exc: if "Please see the 'svn upgrade' command" in str(exc): lgr.warning("SVN at %s is outdated, needs 'svn upgrade'", dirpath) else: # we are in SVN but it is outdated and needs an upgrade lgr.debug("Probably %s is not under SVN repo path: %s", dirpath, exc_str(exc)) return None # for now we treat each directory under SVN independently # pros: # - could provide us 'minimal' set of checkouts to do, since it might be # quite expensive to checkout the entire tree if it was not used # besides few leaves lgr.debug("Detected SVN repository at %s", dirpath) return cls(dirpath, session=session)
def _venv_version(self): try: out, _ = self._session.execute_command(["virtualenv", "--version"]) except Exception as exc: lgr.debug("Could not determine virtualenv version: %s", exc_str(exc)) return return out.strip()
def fetch(self, on_remote_finish=None): """Fetch the results from the remote dataset sibling. Parameters ---------- on_remote_finish : callable, optional Function to be called when work with the resource is finished. It will be passed two arguments, the resource and the failed subjobs (list of ints). """ from datalad.support.exceptions import CommandError as DCError lgr.info("Fetching results for %s", self.jobid) failed = self.get_failed_subjobs() resource_name = self.resource.name ref = self.job_refname lgr.info("Updating local dataset with changes from '%s'", resource_name) repo = self.ds.repo repo.fetch(resource_name, "{0}:{0}".format(ref), recurse_submodules="no") failure = False try: repo.merge(ref) except DCError as exc: lgr.warning( "Failed to merge in changes from %s. " "Check %s for merge conflicts. %s", ref, self.ds.path, exc_str(exc)) failure = True else: # Handle any subdataset updates. We could avoid this if we knew # there were no subdataset changes, but it's probably simplest to # just unconditionally call update(). failure = False try: call_check_dl_results(self.ds.update, "'datalad update' failed", sibling=resource_name, merge=True, follow="parentds", recursive=True, on_failure="ignore") except OrchestratorError: failure = True if not failure: lgr.info("Getting outputs from '%s'", resource_name) outputs = list(self.get_outputs()) if outputs: self.ds.get(path=outputs) self.log_failed(failed, func=lambda mdir, _: self.ds.get(path=mdir)) lgr.info("Finished with remote resource '%s'", resource_name) if on_remote_finish: on_remote_finish(self.resource, failed)
def _is_venv_directory(self, path): try: self._session.execute_command( ["grep", "-q", "VIRTUAL_ENV", "{}/bin/activate".format(path)]) except Exception as exc: lgr.debug("Did not detect virtualenv at the path %s: %s", path, exc_str(exc)) return False return True
def _get_conda_meta_files(self, conda_path): try: out, _ = self._session.execute_command('ls %s/conda-meta/*.json' % conda_path) return iter(out.splitlines()) except Exception as exc: # Empty conda environment (unusual situation) lgr.warning("Could not retrieve conda-meta files in path %s: %s", conda_path, exc_str(exc)) return iter(())
def _get_package_details(self, venv_path): pip = venv_path + "/bin/pip" try: packages, file_to_pkg = piputils.get_package_details( self._session, pip) except Exception as exc: lgr.warning("Could not determine pip package details for %s: %s", venv_path, exc_str(exc)) return {}, {} return packages, file_to_pkg
def __getitem__(self, module): # when ran straight in its source code -- fails to discover nipy's version.. TODO #if module == 'nipy': # import pdb; pdb.set_trace() if not isinstance(module, str): modname = module.__name__ else: modname = module module = None # Early returns None so we do not store prev result for them # and allow users to install things at run time, so later check # doesn't pick it up from the _versions if modname not in self._versions: version = None # by default -- not present if modname in self.CUSTOM: try: version = self.CUSTOM[modname]() version = self._deduce_version(version) except Exception as exc: lgr.debug("Failed to deduce version of %s due to %s" % (modname, exc_str(exc))) return None else: if module is None: if modname not in sys.modules: try: module = __import__(modname) except ImportError: lgr.debug("Module %s seems to be not present" % modname) return None except Exception as exc: lgr.warning("Failed to import module %s due to %s", modname, exc_str(exc)) return None else: module = sys.modules[modname] if module: version = self._deduce_version(module) self._versions[modname] = version return self._versions.get(modname, self.UNKNOWN)
def _get_conda_info(self, conda_path): details = {} try: out, err = self._session.execute_command( '%s/bin/conda info --json' % conda_path) details = json.loads(out) except Exception as exc: lgr.warning("Could not retrieve conda info in path %s: %s", conda_path, exc_str(exc)) return details
def _get_conda_env_export(self, root_prefix, conda_path): export = {} try: # NOTE: We need to call conda-env directly. Conda has problems # calling conda-env without a PATH being set... out, err = self._session.execute_command( '%s/bin/conda-env export -p %s' % (root_prefix, conda_path)) export = yaml.safe_load(out) except Exception as exc: if "unrecognized arguments: -p" in exc_str(exc): lgr.warning( "Could not retrieve conda environment " "export from path %s: " "Please use Conda 4.3.19 or greater", conda_path) else: lgr.warning( "Could not retrieve conda environment " "export from path %s: %s", conda_path, exc_str(exc)) return export
def _python_version(self, venv_path): try: out, err = self._session.execute_command( [venv_path + "/bin/python", "--version"]) # Python 2 sends its version to stderr, while Python 3 # sends it to stdout. pyver = out if "Python" in out else err return pyver.strip().split()[1] except Exception as exc: lgr.debug("Could not determine python version: %s", exc_str(exc)) return
def isdir(self, path): try: out, err = self.execute_command(self.isdir_command(path)) except Exception as exc: # TODO: More specific exception? lgr.debug("Check for directory failed: %s", exc_str(exc)) return False if out == 'Found\n': return True else: lgr.debug("Standard error was not empty (%r), thus assuming that " "test for direcory has failed", err) return False
def exists(self, path): """Return if file exists""" try: out, err = self.execute_command(self.exists_command(path)) except Exception as exc: # TODO: More specific exception? lgr.debug("Check for file presence failed: %s", exc_str(exc)) return False if out == 'Found\n': return True else: lgr.debug("Standard error was not empty (%r), thus assuming that " "test for file presence has failed", err) return False
def get_resource_classes(names=None): for name in names or discover_types(): try: cls = get_resource_class(name) except ResourceError as exc: lgr.warning(exc_str(exc)) continue if issubclass(cls, Resource): yield name, cls else: lgr.debug( "Skipping %s because it is not a Resource. " "Consider moving away", cls)
def _get_system_ssh_version(): """Return version of ssh available system-wide """ try: out, err = _runner.run('ssh -V'.split(), expect_fail=True, expect_stderr=True) # apparently spits out to err but I wouldn't trust it blindly if err.startswith('OpenSSH'): out = err assert out.startswith('OpenSSH') # that is the only one we care about atm return out.split(' ', 1)[0].rstrip(',.').split('_')[1] except CommandError as exc: lgr.debug("Could not determine version of ssh available: %s", exc_str(exc)) return None
def get_at_dirpath(cls, session, dirpath): try: out, err = session.execute_command( 'git rev-parse --show-toplevel', # expect_fail=True, cwd=dirpath) except CommandError as exc: lgr.debug("Probably %s is not under git repo path: %s", dirpath, exc_str(exc)) return None topdir = out.rstrip('\n') lgr.debug( "Detected Git repository at %s for %s. Creating a session shim", topdir, dirpath) return cls(topdir, session=session)
def __call__(queries, action="auto", all_=False, status=False): job_files = LREG.find_job_files() if not job_files: lgr.info("No jobs found") return if all_: matched_ids = job_files.keys() else: matched_ids = [] for query in queries: m = match(query, job_files) if m: matched_ids.append(m) else: lgr.warning("No jobs matched query %s", query) if not matched_ids and action in ["delete", "fetch"]: # These are actions where we don't want to just conveniently # default to "all" unless --all is explicitly specified. raise ValueError("Must specify jobs to {}".format(action)) # We don't need to load the job to delete it, so check that first. if action == "delete": for i in matched_ids: LREG.unregister(i) else: jobs = [_load(job_files[i]) for i in matched_ids or job_files] if action == "fetch" or (action == "auto" and matched_ids): fn = fetch elif action == "list" or action == "auto": fn = partial(show_oneline, status=status) elif action == "show": fn = partial(show, status=status) else: raise RuntimeError("Unknown action: {}".format(action)) for job in jobs: try: fn(job) except OrchestratorError as exc: lgr.error("job %s failed: %s", job["_jobid"], exc_str(exc)) except ResourceNotFoundError: lgr.error("Resource %s (%s) no longer exists", job["resource_id"], job["resource_name"])
def status(self): """Like Orchestrator.status, but inspect the job's git ref if needed. """ status = super(DataladOrchestrator, self).status if status == "unknown": # The local tree might be different because of another just. Check # the ref for the status. try: status_from_ref = self._execute_in_wdir( "git cat-file -p {}:{}" .format(self.job_refname, # FIXME: How to handle subjobs? op.relpath(op.join(self.meta_directory, "status.0"), self.working_directory))) except OrchestratorError as exc: # Most likely the ref was never created because the runscript # failed. Let follow() signal the error. lgr.debug("Failed to get status from %s tree: %s", self.job_refname, exc_str(exc)) else: status = status_from_ref.strip() or status return status
def get_resource_classes(names=None): for name in names or ResourceManager._discover_types(): try: module = import_module('reproman.resource.{}'.format(name)) except ImportError as exc: import difflib known = ResourceManager._discover_types() suggestions = difflib.get_close_matches(name, known) lgr.warning( "Failed to import resource %s: %s. %s: %s", name, exc_str(exc), "Similar backends" if suggestions else "Known backends", ', '.join(suggestions or known)) continue class_name = ''.join([token.capitalize() for token in name.split('_')]) cls = getattr(module, class_name) if issubclass(cls, Resource): yield name, cls else: lgr.debug( "Skipping %s.%s because it is not a Resource. " "Consider moving away", module, class_name)