def _keyring(self):
if self.__keyring_mod is None:
# Setup logging for keyring if we are debugging, although keyring's logging
# is quite scarce ATM
from datalad.log import lgr
import logging
lgr_level = lgr.getEffectiveLevel()
if lgr_level < logging.DEBUG:
keyring_lgr = logging.getLogger('keyring')
keyring_lgr.setLevel(lgr_level)
keyring_lgr.handlers = lgr.handlers
lgr.debug("Importing keyring")
import keyring
self.__keyring_mod = keyring
if self.__keyring is None:
from datalad.log import lgr
# we use module bound interfaces whenever we were not provided a dedicated
# backend
self.__keyring = self.__keyring_mod
the_keyring = self.__keyring_mod.get_keyring()
if the_keyring.name.lower().startswith('null '):
lgr.warning(
"Keyring module returned '%s', no credentials will be provided",
the_keyring.name)
return self.__keyring
def add(self, name, func=None):
"""Add a version checker
This method allows third-party libraries to define additional checks.
It will not add `name` if already exists. If `name` exists and `func`
is different - it will override with a new `func`. Added entries will
be included in the output of `dumps(query=True)`.
Parameters
----------
name: str
Name of the check (usually a name of the Python module, or an
external command prefixed with "cmd:")
func: callable, optional
Function to be called to obtain version information. This should be
defined when checking the version of something that is not a Python
module or when this class's method for determining the version of a
Python module isn't sufficient.
"""
if func:
func_existing = self.CUSTOM.get(name, None)
was_known = False
if func_existing and func_existing is not func:
lgr.debug(
"Adding a new custom version checker %s for %s, "
"old one: %s", func, name, func_existing)
was_known = name in self._versions
self.CUSTOM[name] = func
if was_known:
# pop and query it again right away to possibly replace with a new value
self._versions.pop(name)
_ = self[name]
elif name not in self.INTERESTING:
self.INTERESTING.append(name)
def run(args):
from datalad.api import Dataset
from datalad.log import lgr
lgr.debug("Command line arguments: %r" % args)
path = os.path.expandvars(os.path.expanduser(args.destination))
ds = Dataset(path, args.source, direct=args.direct)
def run(args):
from datalad.api import Dataset
from datalad.log import lgr
lgr.debug("Command line arguments: %r" % args)
path = os.path.expandvars(os.path.expanduser(args.destination))
ds = Dataset(path, args.source, direct=args.direct)
def teardown_package():
from datalad.tests import _TEMP_PATHS_GENERATED
from datalad.tests.utils import rmtemp
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
def setup_package():
import os
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
def teardown_package():
from datalad.tests import _TEMP_PATHS_GENERATED
from datalad.tests.utils import rmtemp
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
def _get_system_7z_version():
"""Return version of 7-Zip"""
out = _runner.run(['7z'], protocol=StdOutErrCapture)
# reporting in variable order across platforms
# Linux: 7-Zip [64] 16.02
# Windows: 7-Zip 19.00 (x86)
pieces = out['stdout'].strip().split(':', maxsplit=1)[0].strip().split()
for p in pieces:
# the one with the dot is the version
if '.' in p:
return p
lgr.debug("Could not determine version of 7z from stdout. %s", out)
def run(args):
from datalad.api import DoubleAnnexRepo, load_config
from datalad.log import lgr
lgr.debug("Command line arguments: %r" % args)
lgr.info("Reading configs")
cfg = load_config(args.configs)
drepo = DoubleAnnexRepo(cfg)
drepo.page2annex(existing=args.existing, dry_run=args.dry_run,
cache=args.cache)
def run(args):
from datalad.api import DoubleAnnexRepo, load_config
from datalad.log import lgr
lgr.debug("Command line arguments: %r" % args)
lgr.info("Reading configs")
cfg = load_config(args.configs)
drepo = DoubleAnnexRepo(cfg)
drepo.page2annex(existing=args.existing,
dry_run=args.dry_run,
cache=args.cache)
def _get_system_7z_version():
"""Return version of 7-Zip"""
out, err = _runner.run(
['7z'], expect_fail=True, expect_stderr=True,
)
# reporting in variable order across platforms
# Linux: 7-Zip [64] 16.02
# Windows: 7-Zip 19.00 (x86)
pieces = out.strip().split(':', maxsplit=1)[0].strip().split()
for p in pieces:
# the one with the dot is the version
if '.' in p:
return p
lgr.debug("Could not determine version of 7z from stdout. "
"stdout: %s, stderr: %s", out, err)
def setup_package():
import os
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
def _keyring(self):
if self.__keyring is None:
# Setup logging for keyring if we are debugging, althought keyring's logging
# is quite scarce ATM
from datalad.log import lgr
import logging
lgr_level = lgr.getEffectiveLevel()
if lgr_level < logging.DEBUG:
keyring_lgr = logging.getLogger('keyring')
keyring_lgr.setLevel(lgr_level)
keyring_lgr.handlers = lgr.handlers
lgr.debug("Importing keyring")
import keyring
self.__keyring = keyring
return self.__keyring
def _keyring(self):
if self.__keyring is None:
# Setup logging for keyring if we are debugging, althought keyring's logging
# is quite scarce ATM
from datalad.log import lgr
import logging
lgr_level = lgr.getEffectiveLevel()
if lgr_level < logging.DEBUG:
keyring_lgr = logging.getLogger('keyring')
keyring_lgr.setLevel(lgr_level)
keyring_lgr.handlers = lgr.handlers
lgr.debug("Importing keyring")
import keyring
self.__keyring = keyring
return self.__keyring
def setup_package():
import os
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
def _get_system_ssh_version():
"""Return version of ssh available system-wide
Annex prior 20170302 was using bundled version, but now would use system one
if installed
"""
try:
out, err = _runner.run('ssh -V'.split(),
expect_fail=True, expect_stderr=True)
# apparently spits out to err but I wouldn't trust it blindly
if err.startswith('OpenSSH'):
out = err
assert out.startswith('OpenSSH') # that is the only one we care about atm
return out.split(' ', 1)[0].rstrip(',.').split('_')[1]
except CommandError as exc:
lgr.debug("Could not determine version of ssh available: %s", exc_str(exc))
return None
def __getitem__(self, module):
# when ran straight in its source code -- fails to discover nipy's version.. TODO
#if module == 'nipy':
# import pdb; pdb.set_trace()
if not isinstance(module, str):
modname = module.__name__
else:
modname = module
module = None
lgr.log(5, "Requested to provide version for %s", modname)
# Early returns None so we do not store prev result for them
# and allow users to install things at run time, so later check
# doesn't pick it up from the _versions
if modname not in self._versions:
version = None # by default -- not present
if modname in self.CUSTOM:
try:
version = self.CUSTOM[modname]()
version = self._deduce_version(version)
except Exception as exc:
lgr.debug("Failed to deduce version of %s due to %s" %
(modname, exc_str(exc)))
return None
else:
if module is None:
if modname not in sys.modules:
try:
module = __import__(modname)
except ImportError:
lgr.debug("Module %s seems to be not present" %
modname)
return None
except Exception as exc:
lgr.warning("Failed to import module %s due to %s",
modname, exc_str(exc))
return None
else:
module = sys.modules[modname]
if module:
version = self._deduce_version(module)
self._versions[modname] = version
return self._versions.get(modname, self.UNKNOWN)
def run(args):
import glob
import os
import os.path
from datalad.api import Dataset
from datalad.log import lgr
lgr.debug("Command line arguments: %r" % args)
# Since GitPython doesn't recognize we ar with in a repo, if we are
# deeper down the tree, walk upwards and look for '.git':
# TODO: May be provide a patch for GitPython to have it cleaner.
cwd_before = cwd = os.getcwd()
while True:
if os.path.exists(os.path.join(cwd, '.git')):
break
else:
if cwd == '/': # TODO: Is this platform-dependend?
lgr.error("No repository found.")
raise ValueError # TODO: Proper Exception or clean exit?
else:
os.chdir(os.pardir)
cwd = os.getcwd()
ds = Dataset(cwd)
os.chdir(cwd_before)
# args.path comes as a list
# Expansions (like globs) provided by the shell itself are already done.
# But: We don't know exactly what shells we are running on and what it may provide or not.
# Therefore, make any expansion we want to guarantee, per item of the list:
expanded_list = []
for item in args.path:
expanded_list.extend(glob.glob(item))
# TODO: regexp + may be ext. glob zsh-style
# TODO: what about spaces in filenames and similar things?
# TODO: os.path.expandvars, os.path.expanduser? Is not needed here, isn't it? Always?
ds.get(expanded_list)
def get_metadata_type(ds, guess=False):
"""Return the metadata type(s)/scheme(s) of a dataset
Parameters
----------
ds : Dataset
Dataset instance to be inspected
guess : bool
Whether to try to auto-detect the type if no metadata type setting is
found. All supported metadata schemes are tested in alphanumeric order.
Returns
-------
list(str)
Metadata type labels or an empty list if no type setting is found and
optional auto-detection yielded no results
"""
cfg_ = ds.config
# TODO give cfg name datalad prefix
if cfg_ and cfg_.has_section('metadata'):
if cfg_.has_option('metadata', 'nativetype'):
return cfg_.get_value('metadata', 'nativetype').split()
mtypes = []
if guess:
# keep local, who knows what some parsers might pull in
from . import parsers
for mtype in sorted([
p for p in parsers.__dict__
if not (p.startswith('_') or p in ('tests', 'base'))
]):
if mtype == 'aggregate':
# skip, runs anyway, but later
continue
pmod = import_module('.%s' % (mtype, ),
package=parsers.__package__)
if pmod.MetadataParser(ds).has_metadata():
lgr.debug('Predicted presence of "%s" meta data', mtype)
mtypes.append(mtype)
else:
lgr.debug('No evidence for "%s" meta data', mtype)
return mtypes
def __getitem__(self, module):
# when ran straight in its source code -- fails to discover nipy's version.. TODO
#if module == 'nipy':
# import pdb; pdb.set_trace()
if not isinstance(module, string_types):
modname = module.__name__
else:
modname = module
module = None
# Early returns None so we do not store prev result for them
# and allow users to install things at run time, so later check
# doesn't pick it up from the _versions
if modname not in self._versions:
version = None # by default -- not present
if modname in self.CUSTOM:
try:
version = self.CUSTOM[modname]()
version = self._deduce_version(version)
except Exception as exc:
lgr.debug("Failed to deduce version of %s due to %s"
% (modname, exc_str(exc)))
return None
else:
if module is None:
if modname not in sys.modules:
try:
module = __import__(modname)
except ImportError:
lgr.debug("Module %s seems to be not present" % modname)
return None
else:
module = sys.modules[modname]
if module:
version = self._deduce_version(module)
self._versions[modname] = version
return self._versions.get(modname, self.UNKNOWN)
def _cached_load_document(url):
"""Loader of pyld document from a url, which caches loaded instance on disk
"""
doc_fname = _get_schema_url_cache_filename(url)
doc = None
if os.path.exists(doc_fname):
try:
lgr.debug("use cached request result to '%s' from %s", url, doc_fname)
doc = pickle.load(open(doc_fname, 'rb'))
except Exception as e: # it is OK to ignore any error and fall back on the true source
lgr.warning(
"cannot load cache from '%s', fall back on schema download: %s",
doc_fname, exc_str(e))
if doc is None:
from pyld.jsonld import load_document
doc = load_document(url)
assure_dir(dirname(doc_fname))
# use pickle to store the entire request result dict
pickle.dump(doc, open(doc_fname, 'wb'))
lgr.debug("stored result of request to '{}' in {}".format(url, doc_fname))
return doc
def get_metadata_type(ds, guess=False):
"""Return the metadata type(s)/scheme(s) of a dataset
Parameters
----------
ds : Dataset
Dataset instance to be inspected
guess : bool
Whether to try to auto-detect the type if no metadata type setting is
found. All supported metadata schemes are tested in alphanumeric order.
Returns
-------
list(str)
Metadata type labels or an empty list if no type setting is found and
optional auto-detection yielded no results
"""
cfg_ = ds.config
# TODO give cfg name datalad prefix
if cfg_ and cfg_.has_section('metadata'):
if cfg_.has_option('metadata', 'nativetype'):
return cfg_.get_value('metadata', 'nativetype').split()
mtypes = []
if guess:
# keep local, who knows what some parsers might pull in
from . import parsers
for mtype in sorted([p for p in parsers.__dict__ if not (p.startswith('_') or p in ('tests', 'base'))]):
if mtype == 'aggregate':
# skip, runs anyway, but later
continue
pmod = import_module('.%s' % (mtype,), package=parsers.__package__)
if pmod.MetadataParser(ds).has_metadata():
lgr.debug('Predicted presence of "%s" meta data', mtype)
mtypes.append(mtype)
else:
lgr.debug('No evidence for "%s" meta data', mtype)
return mtypes
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the datalad package for the
# copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""DataLad aims to expose (scientific) data available online as a unified data
distribution with the convenience of git-annex repositories as a backend."""
from .version import __version__
from datalad.log import lgr
lgr.debug("Importing the rest of datalad.__init__")
from .config import ConfigManager
cfg = ConfigManager()
# be friendly on systems with ancient numpy -- no tests, but at least
# importable
try:
from numpy.testing import Tester
test = Tester().test
bench = Tester().bench
del Tester
except ImportError:
def test(*args, **kwargs):
lgr.warning('Need numpy >= 1.2 for datalad.tests(). Nothing is done')
test.__test__ = False
# Following fixtures are necessary at the top level __init__ for fixtures which
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the datalad package for the
# copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""DataLad aims to expose (scientific) data available online as a unified data
distribution with the convenience of git-annex repositories as a backend."""
from .version import __version__
from datalad.log import lgr
lgr.debug("Importing the rest of datalad.__init__")
from .config import ConfigManager
cfg = ConfigManager()
# be friendly on systems with ancient numpy -- no tests, but at least
# importable
try:
from numpy.testing import Tester
test = Tester().test
bench = Tester().bench
del Tester
except ImportError:
def test(*args, **kwargs):
lgr.warning('Need numpy >= 1.2 for datalad.tests(). Nothing is done')
test.__test__ = False