Ejemplo n.º 1
0
    def _keyring(self):
        if self.__keyring_mod is None:
            # Setup logging for keyring if we are debugging, although keyring's logging
            # is quite scarce ATM
            from datalad.log import lgr
            import logging
            lgr_level = lgr.getEffectiveLevel()
            if lgr_level < logging.DEBUG:
                keyring_lgr = logging.getLogger('keyring')
                keyring_lgr.setLevel(lgr_level)
                keyring_lgr.handlers = lgr.handlers
            lgr.debug("Importing keyring")
            import keyring
            self.__keyring_mod = keyring

        if self.__keyring is None:
            from datalad.log import lgr
            # we use module bound interfaces whenever we were not provided a dedicated
            # backend
            self.__keyring = self.__keyring_mod
            the_keyring = self.__keyring_mod.get_keyring()
            if the_keyring.name.lower().startswith('null '):
                lgr.warning(
                    "Keyring module returned '%s', no credentials will be provided",
                    the_keyring.name)
        return self.__keyring
Ejemplo n.º 2
0
    def add(self, name, func=None):
        """Add a version checker

        This method allows third-party libraries to define additional checks.
        It will not add `name` if already exists.  If `name` exists and `func`
        is different - it will override with a new `func`.  Added entries will
        be included in the output of `dumps(query=True)`.

        Parameters
        ----------
        name: str
          Name of the check (usually a name of the Python module, or an
          external command prefixed with "cmd:")
        func: callable, optional
          Function to be called to obtain version information. This should be
          defined when checking the version of something that is not a Python
          module or when this class's method for determining the version of a
          Python module isn't sufficient.
        """
        if func:
            func_existing = self.CUSTOM.get(name, None)
            was_known = False
            if func_existing and func_existing is not func:
                lgr.debug(
                    "Adding a new custom version checker %s for %s, "
                    "old one: %s", func, name, func_existing)
                was_known = name in self._versions
            self.CUSTOM[name] = func
            if was_known:
                # pop and query it again right away to possibly replace with a new value
                self._versions.pop(name)
                _ = self[name]
        elif name not in self.INTERESTING:
            self.INTERESTING.append(name)
Ejemplo n.º 3
0
def run(args):
    from datalad.api import Dataset
    from datalad.log import lgr

    lgr.debug("Command line arguments: %r" % args)

    path = os.path.expandvars(os.path.expanduser(args.destination))
    ds = Dataset(path, args.source, direct=args.direct)
Ejemplo n.º 4
0
def run(args):
    from datalad.api import Dataset
    from datalad.log import lgr

    lgr.debug("Command line arguments: %r" % args)

    path = os.path.expandvars(os.path.expanduser(args.destination))
    ds = Dataset(path, args.source, direct=args.direct)
Ejemplo n.º 5
0
def teardown_package():
    from datalad.tests import _TEMP_PATHS_GENERATED
    from datalad.tests.utils import rmtemp
    if len(_TEMP_PATHS_GENERATED):
        msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
    else:
        msg = "Nothing to remove"
    lgr.debug("Teardown tests. " + msg)
    for path in _TEMP_PATHS_GENERATED:
        rmtemp(path, ignore_errors=True)
Ejemplo n.º 6
0
def setup_package():
    import os
    # To overcome pybuild by default defining http{,s}_proxy we would need
    # to define them to e.g. empty value so it wouldn't bother touching them.
    # But then haskell libraries do not digest empty value nicely, so we just
    # pop them out from the environment
    for ev in ('http_proxy', 'https_proxy'):
        if ev in os.environ and not (os.environ[ev]):
            lgr.debug("Removing %s from the environment since it is empty", ev)
            os.environ.pop(ev)
Ejemplo n.º 7
0
def teardown_package():
    from datalad.tests import _TEMP_PATHS_GENERATED
    from datalad.tests.utils import rmtemp
    if len(_TEMP_PATHS_GENERATED):
        msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
                                              ', '.join(_TEMP_PATHS_GENERATED))
    else:
        msg = "Nothing to remove"
    lgr.debug("Teardown tests. " + msg)
    for path in _TEMP_PATHS_GENERATED:
        rmtemp(path, ignore_errors=True)
Ejemplo n.º 8
0
def _get_system_7z_version():
    """Return version of 7-Zip"""
    out = _runner.run(['7z'], protocol=StdOutErrCapture)
    # reporting in variable order across platforms
    # Linux: 7-Zip [64] 16.02
    # Windows: 7-Zip 19.00 (x86)
    pieces = out['stdout'].strip().split(':', maxsplit=1)[0].strip().split()
    for p in pieces:
        # the one with the dot is the version
        if '.' in p:
            return p
    lgr.debug("Could not determine version of 7z from stdout. %s", out)
Ejemplo n.º 9
0
def run(args):
    from datalad.api import DoubleAnnexRepo, load_config
    from datalad.log import lgr

    lgr.debug("Command line arguments: %r" % args)

    lgr.info("Reading configs")
    cfg = load_config(args.configs)

    drepo = DoubleAnnexRepo(cfg)
    drepo.page2annex(existing=args.existing, dry_run=args.dry_run,
                     cache=args.cache)
Ejemplo n.º 10
0
def run(args):
    from datalad.api import DoubleAnnexRepo, load_config
    from datalad.log import lgr

    lgr.debug("Command line arguments: %r" % args)

    lgr.info("Reading configs")
    cfg = load_config(args.configs)

    drepo = DoubleAnnexRepo(cfg)
    drepo.page2annex(existing=args.existing,
                     dry_run=args.dry_run,
                     cache=args.cache)
Ejemplo n.º 11
0
def _get_system_7z_version():
    """Return version of 7-Zip"""
    out, err = _runner.run(
        ['7z'], expect_fail=True, expect_stderr=True,
    )
    # reporting in variable order across platforms
    # Linux: 7-Zip [64] 16.02
    # Windows: 7-Zip 19.00 (x86)
    pieces = out.strip().split(':', maxsplit=1)[0].strip().split()
    for p in pieces:
        # the one with the dot is the version
        if '.' in p:
            return p
    lgr.debug("Could not determine version of 7z from stdout. "
              "stdout: %s, stderr: %s", out, err)
Ejemplo n.º 12
0
def setup_package():
    import os

    # To overcome pybuild overriding HOME but us possibly wanting our
    # own HOME where we pre-setup git for testing (name, email)
    if 'GIT_HOME' in os.environ:
        os.environ['HOME'] = os.environ['GIT_HOME']

    # To overcome pybuild by default defining http{,s}_proxy we would need
    # to define them to e.g. empty value so it wouldn't bother touching them.
    # But then haskell libraries do not digest empty value nicely, so we just
    # pop them out from the environment
    for ev in ('http_proxy', 'https_proxy'):
        if ev in os.environ and not (os.environ[ev]):
            lgr.debug("Removing %s from the environment since it is empty", ev)
            os.environ.pop(ev)
Ejemplo n.º 13
0
    def _keyring(self):
        if self.__keyring is None:
            # Setup logging for keyring if we are debugging, althought keyring's logging
            # is quite scarce ATM
            from datalad.log import lgr
            import logging
            lgr_level = lgr.getEffectiveLevel()
            if lgr_level < logging.DEBUG:
                keyring_lgr = logging.getLogger('keyring')
                keyring_lgr.setLevel(lgr_level)
                keyring_lgr.handlers = lgr.handlers
            lgr.debug("Importing keyring")
            import keyring
            self.__keyring = keyring

        return self.__keyring
Ejemplo n.º 14
0
    def _keyring(self):
        if self.__keyring is None:
            # Setup logging for keyring if we are debugging, althought keyring's logging
            # is quite scarce ATM
            from datalad.log import lgr
            import logging
            lgr_level = lgr.getEffectiveLevel()
            if lgr_level < logging.DEBUG:
                keyring_lgr = logging.getLogger('keyring')
                keyring_lgr.setLevel(lgr_level)
                keyring_lgr.handlers = lgr.handlers
            lgr.debug("Importing keyring")
            import keyring
            self.__keyring = keyring

        return self.__keyring
Ejemplo n.º 15
0
def setup_package():
    import os

    # To overcome pybuild overriding HOME but us possibly wanting our
    # own HOME where we pre-setup git for testing (name, email)
    if 'GIT_HOME' in os.environ:
        os.environ['HOME'] = os.environ['GIT_HOME']

    # To overcome pybuild by default defining http{,s}_proxy we would need
    # to define them to e.g. empty value so it wouldn't bother touching them.
    # But then haskell libraries do not digest empty value nicely, so we just
    # pop them out from the environment
    for ev in ('http_proxy', 'https_proxy'):
        if ev in os.environ and not (os.environ[ev]):
            lgr.debug("Removing %s from the environment since it is empty", ev)
            os.environ.pop(ev)
Ejemplo n.º 16
0
def _get_system_ssh_version():
    """Return version of ssh available system-wide

    Annex prior 20170302 was using bundled version, but now would use system one
    if installed
    """
    try:
        out, err = _runner.run('ssh -V'.split(),
                               expect_fail=True, expect_stderr=True)
        # apparently spits out to err but I wouldn't trust it blindly
        if err.startswith('OpenSSH'):
            out = err
        assert out.startswith('OpenSSH')  # that is the only one we care about atm
        return out.split(' ', 1)[0].rstrip(',.').split('_')[1]
    except CommandError as exc:
        lgr.debug("Could not determine version of ssh available: %s", exc_str(exc))
        return None
Ejemplo n.º 17
0
    def __getitem__(self, module):
        # when ran straight in its source code -- fails to discover nipy's version.. TODO
        #if module == 'nipy':
        #    import pdb; pdb.set_trace()
        if not isinstance(module, str):
            modname = module.__name__
        else:
            modname = module
            module = None

        lgr.log(5, "Requested to provide version for %s", modname)
        # Early returns None so we do not store prev result for  them
        # and allow users to install things at run time, so later check
        # doesn't pick it up from the _versions
        if modname not in self._versions:
            version = None  # by default -- not present
            if modname in self.CUSTOM:
                try:
                    version = self.CUSTOM[modname]()
                    version = self._deduce_version(version)
                except Exception as exc:
                    lgr.debug("Failed to deduce version of %s due to %s" %
                              (modname, exc_str(exc)))
                    return None
            else:
                if module is None:
                    if modname not in sys.modules:
                        try:
                            module = __import__(modname)
                        except ImportError:
                            lgr.debug("Module %s seems to be not present" %
                                      modname)
                            return None
                        except Exception as exc:
                            lgr.warning("Failed to import module %s due to %s",
                                        modname, exc_str(exc))
                            return None
                    else:
                        module = sys.modules[modname]
                if module:
                    version = self._deduce_version(module)
            self._versions[modname] = version

        return self._versions.get(modname, self.UNKNOWN)
Ejemplo n.º 18
0
def run(args):
    import glob
    import os
    import os.path

    from datalad.api import Dataset
    from datalad.log import lgr

    lgr.debug("Command line arguments: %r" % args)

    # Since GitPython doesn't recognize we ar with in a repo, if we are
    # deeper down the tree, walk upwards and look for '.git':
    # TODO: May be provide a patch for GitPython to have it cleaner.
    cwd_before = cwd = os.getcwd()
    while True:
        if os.path.exists(os.path.join(cwd, '.git')):
            break
        else:
            if cwd == '/':  # TODO: Is this platform-dependend?
                lgr.error("No repository found.")
                raise ValueError  # TODO: Proper Exception or clean exit?
            else:
                os.chdir(os.pardir)
                cwd = os.getcwd()

    ds = Dataset(cwd)
    os.chdir(cwd_before)

    # args.path comes as a list
    # Expansions (like globs) provided by the shell itself are already done.
    # But: We don't know exactly what shells we are running on and what it may provide or not.
    # Therefore, make any expansion we want to guarantee, per item of the list:

    expanded_list = []
    for item in args.path:
        expanded_list.extend(glob.glob(item))
        # TODO: regexp + may be ext. glob zsh-style
        # TODO: what about spaces in filenames and similar things?
        # TODO: os.path.expandvars, os.path.expanduser? Is not needed here, isn't it? Always?

    ds.get(expanded_list)
Ejemplo n.º 19
0
def get_metadata_type(ds, guess=False):
    """Return the metadata type(s)/scheme(s) of a dataset

    Parameters
    ----------
    ds : Dataset
      Dataset instance to be inspected
    guess : bool
      Whether to try to auto-detect the type if no metadata type setting is
      found. All supported metadata schemes are tested in alphanumeric order.

    Returns
    -------
    list(str)
      Metadata type labels or an empty list if no type setting is found and
      optional auto-detection yielded no results
    """
    cfg_ = ds.config
    # TODO give cfg name datalad prefix
    if cfg_ and cfg_.has_section('metadata'):
        if cfg_.has_option('metadata', 'nativetype'):
            return cfg_.get_value('metadata', 'nativetype').split()
    mtypes = []
    if guess:
        # keep local, who knows what some parsers might pull in
        from . import parsers
        for mtype in sorted([
                p for p in parsers.__dict__
                if not (p.startswith('_') or p in ('tests', 'base'))
        ]):
            if mtype == 'aggregate':
                # skip, runs anyway, but later
                continue
            pmod = import_module('.%s' % (mtype, ),
                                 package=parsers.__package__)
            if pmod.MetadataParser(ds).has_metadata():
                lgr.debug('Predicted presence of "%s" meta data', mtype)
                mtypes.append(mtype)
            else:
                lgr.debug('No evidence for "%s" meta data', mtype)
    return mtypes
Ejemplo n.º 20
0
    def __getitem__(self, module):
        # when ran straight in its source code -- fails to discover nipy's version.. TODO
        #if module == 'nipy':
        #    import pdb; pdb.set_trace()
        if not isinstance(module, string_types):
            modname = module.__name__
        else:
            modname = module
            module = None

        # Early returns None so we do not store prev result for  them
        # and allow users to install things at run time, so later check
        # doesn't pick it up from the _versions
        if modname not in self._versions:
            version = None   # by default -- not present
            if modname in self.CUSTOM:
                try:
                    version = self.CUSTOM[modname]()
                    version = self._deduce_version(version)
                except Exception as exc:
                    lgr.debug("Failed to deduce version of %s due to %s"
                              % (modname, exc_str(exc)))
                    return None
            else:
                if module is None:
                    if modname not in sys.modules:
                        try:
                            module = __import__(modname)
                        except ImportError:
                            lgr.debug("Module %s seems to be not present" % modname)
                            return None
                    else:
                        module = sys.modules[modname]
                if module:
                    version = self._deduce_version(module)
            self._versions[modname] = version

        return self._versions.get(modname, self.UNKNOWN)
Ejemplo n.º 21
0
def _cached_load_document(url):
    """Loader of pyld document from a url, which caches loaded instance on disk
    """
    doc_fname = _get_schema_url_cache_filename(url)

    doc = None
    if os.path.exists(doc_fname):
        try:
            lgr.debug("use cached request result to '%s' from %s", url, doc_fname)
            doc = pickle.load(open(doc_fname, 'rb'))
        except Exception as e:  # it is OK to ignore any error and fall back on the true source
            lgr.warning(
                "cannot load cache from '%s', fall back on schema download: %s",
                doc_fname, exc_str(e))

    if doc is None:
        from pyld.jsonld import load_document
        doc = load_document(url)
        assure_dir(dirname(doc_fname))
        # use pickle to store the entire request result dict
        pickle.dump(doc, open(doc_fname, 'wb'))
        lgr.debug("stored result of request to '{}' in {}".format(url, doc_fname))
    return doc
Ejemplo n.º 22
0
def get_metadata_type(ds, guess=False):
    """Return the metadata type(s)/scheme(s) of a dataset

    Parameters
    ----------
    ds : Dataset
      Dataset instance to be inspected
    guess : bool
      Whether to try to auto-detect the type if no metadata type setting is
      found. All supported metadata schemes are tested in alphanumeric order.

    Returns
    -------
    list(str)
      Metadata type labels or an empty list if no type setting is found and
      optional auto-detection yielded no results
    """
    cfg_ = ds.config
    # TODO give cfg name datalad prefix
    if cfg_ and cfg_.has_section('metadata'):
        if cfg_.has_option('metadata', 'nativetype'):
            return cfg_.get_value('metadata', 'nativetype').split()
    mtypes = []
    if guess:
        # keep local, who knows what some parsers might pull in
        from . import parsers
        for mtype in sorted([p for p in parsers.__dict__ if not (p.startswith('_') or p in ('tests', 'base'))]):
            if mtype == 'aggregate':
                # skip, runs anyway, but later
                continue
            pmod = import_module('.%s' % (mtype,), package=parsers.__package__)
            if pmod.MetadataParser(ds).has_metadata():
                lgr.debug('Predicted presence of "%s" meta data', mtype)
                mtypes.append(mtype)
            else:
                lgr.debug('No evidence for "%s" meta data', mtype)
    return mtypes
Ejemplo n.º 23
0
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""DataLad aims to expose (scientific) data available online as a unified data
distribution with the convenience of git-annex repositories as a backend."""

from .version import __version__

from datalad.log import lgr
lgr.debug("Importing the rest of datalad.__init__")
from .config import ConfigManager
cfg = ConfigManager()

# be friendly on systems with ancient numpy -- no tests, but at least
# importable
try:
    from numpy.testing import Tester
    test = Tester().test
    bench = Tester().bench
    del Tester
except ImportError:
    def test(*args, **kwargs):
        lgr.warning('Need numpy >= 1.2 for datalad.tests().  Nothing is done')
    test.__test__ = False

# Following fixtures are necessary at the top level __init__ for fixtures which
Ejemplo n.º 24
0
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""DataLad aims to expose (scientific) data available online as a unified data
distribution with the convenience of git-annex repositories as a backend."""

from .version import __version__

from datalad.log import lgr
lgr.debug("Importing the rest of datalad.__init__")
from .config import ConfigManager
cfg = ConfigManager()

# be friendly on systems with ancient numpy -- no tests, but at least
# importable
try:
    from numpy.testing import Tester
    test = Tester().test
    bench = Tester().bench
    del Tester
except ImportError:

    def test(*args, **kwargs):
        lgr.warning('Need numpy >= 1.2 for datalad.tests().  Nothing is done')

    test.__test__ = False