def test_from_env():
cfg = ConfigManager()
assert_not_in('datalad.crazy.cfg', cfg)
os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess'
cfg.reload()
assert_in('datalad.crazy.cfg', cfg)
assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess')
# not in dataset-only mode
cfg = ConfigManager(Dataset('nowhere'), dataset_only=True)
assert_not_in('datalad.crazy.cfg', cfg)
def test_crazy_cfg(path):
cfg = ConfigManager(Dataset(opj(path, 'ds')), source='dataset')
assert_in('crazy.padry', cfg)
# make sure crazy config is not read when in local mode
cfg = ConfigManager(Dataset(opj(path, 'ds')), source='local')
assert_not_in('crazy.padry', cfg)
# it will make it in in 'any' mode though
cfg = ConfigManager(Dataset(opj(path, 'ds')), source='any')
assert_in('crazy.padry', cfg)
# typos in the source mode arg will not have silent side-effects
assert_raises(
ValueError, ConfigManager, Dataset(opj(path, 'ds')), source='locale')
def test_inherit_src_candidates(lcl, storepath, url):
lcl = Path(lcl)
storepath = Path(storepath)
# dataset with a subdataset
ds1 = Dataset(lcl / 'ds1').create()
ds1sub = ds1.create('sub')
# a different dataset into which we install ds1, but do not touch its subds
ds2 = Dataset(lcl / 'ds2').create()
ds2.clone(source=ds1.path, path='mysub')
# we give no dataset a source candidate config!
# move all dataset into the store
for d in (ds1, ds1sub, ds2):
_move2store(storepath, d)
# now we must be able to obtain all three datasets from the store
riaclone = clone(
'ria+{}#{}'.format(
# store URL
url,
# ID of the root dataset
ds2.id),
lcl / 'clone',
)
# what happens is the the initial clone call sets a source candidate
# config, because it sees the dataset coming from a store
# all obtained subdatasets get the config inherited on-clone
datasets = riaclone.get('.', get_data=False, recursive=True, result_xfm='datasets')
# we get two subdatasets
eq_(len(datasets), 2)
for ds in datasets:
eq_(ConfigManager(dataset=ds, source='dataset-local').get(
'datalad.get.subdataset-source-candidate-200origin'),
'ria+%s#{id}' % url)
def config(self):
"""Get an instance of the parser for the persistent dataset configuration.
Note, that this property is evaluated every time it is used. If used
multiple times within a function it's probably a good idea to store its
value in a local variable and use this variable instead.
Returns
-------
ConfigManager
"""
if self.repo is None:
# if there's no repo (yet or anymore), we can't read/write config at
# dataset level, but only at user/system level
# However, if this was the case before as well, we don't want a new
# instance of ConfigManager
if self._cfg_bound in (True, None):
self._cfg = ConfigManager(dataset=None, dataset_only=False)
self._cfg_bound = False
else:
self._cfg = self.repo.config
self._cfg_bound = True
return self._cfg
def config(self):
"""Get an instance of the parser for the persistent dataset configuration.
Returns
-------
ConfigManager
"""
if self._cfg is None:
# associate with this dataset and read the entire config hierarchy
self._cfg = ConfigManager(dataset=self, dataset_only=False)
return self._cfg
def cfg(self):
"""Get a ConfigManager instance for this repository
Returns
-------
ConfigManager
"""
if self._cfg is None:
# associate with this dataset and read the entire config hierarchy
self._cfg = ConfigManager(dataset=self, source='any')
return self._cfg
def test_dataset_local_mode(path=None):
ds = create(path)
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', ds.config)
# from .datalad/config
assert_in('datalad.dataset.id', ds.config)
# from .git/config
assert_in('annex.version', ds.config)
# now check that dataset-local mode doesn't have the global piece
cfg = ConfigManager(ds, source='branch-local')
assert_not_in('user.name', cfg)
assert_in('datalad.dataset.id', cfg)
assert_in('annex.version', cfg)
class CommitInfo():
"""Context manager for setting commit info on datalad operations that use it."""
def __init__(self, dataset, name=None, email=None, where='local'):
self.config_manager = ConfigManager(dataset)
self.email = email if email else SERVICE_EMAIL
self.name = name if name else SERVICE_USER
self.where = where
def __enter__(self):
self.config_manager.set('user.email', self.email, self.where)
self.config_manager.set('user.name', self.name, self.where)
def __exit__(self, exception_type, exception_value, traceback):
self.config_manager.set('user.email', SERVICE_EMAIL, self.where)
self.config_manager.set('user.name', SERVICE_USER, self.where)
def test_from_env():
cfg = ConfigManager()
assert_not_in('datalad.crazy.cfg', cfg)
os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess'
cfg.reload()
assert_in('datalad.crazy.cfg', cfg)
assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess')
# not in dataset-only mode
cfg = ConfigManager(Dataset('nowhere'), dataset_only=True)
assert_not_in('datalad.crazy.cfg', cfg)
# check env trumps override
cfg = ConfigManager()
assert_not_in('datalad.crazy.override', cfg)
cfg.overrides['datalad.crazy.override'] = 'fromoverride'
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromoverride')
os.environ['DATALAD_CRAZY_OVERRIDE'] = 'fromenv'
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_dataset_systemglobal_mode(path=None):
ds = create(path)
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', ds.config)
# from .datalad/config
assert_in('datalad.dataset.id', ds.config)
# from .git/config
assert_in('annex.version', ds.config)
with chpwd(path):
# now check that no config from a random dataset at PWD is picked up
# if not dataset instance was provided
cfg = ConfigManager(dataset=None, source='any')
assert_in('user.name', cfg)
assert_not_in('datalad.dataset.id', cfg)
assert_not_in('annex.version', cfg)
def test_from_env():
cfg = ConfigManager()
assert_not_in('datalad.crazy.cfg', cfg)
os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess'
cfg.reload()
assert_in('datalad.crazy.cfg', cfg)
assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess')
# not in dataset-only mode
cfg = ConfigManager(Dataset('nowhere'), dataset_only=True)
assert_not_in('datalad.crazy.cfg', cfg)
def test_no_local_write_if_no_dataset(path=None):
Dataset(path).create()
with chpwd(path):
cfg = ConfigManager()
with assert_raises(CommandError):
cfg.set('a.b.c', 'd', scope='local')
def test_overrides():
cfg = ConfigManager()
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', cfg)
# set
cfg.set('user.name', 'myoverride', scope='override')
assert_equal(cfg['user.name'], 'myoverride')
# unset just removes override, not entire config
cfg.unset('user.name', scope='override')
assert_in('user.name', cfg)
assert_not_equal('user.name', 'myoverride')
# add
# there is no initial increment
cfg.add('user.name', 'myoverride', scope='override')
assert_equal(cfg['user.name'], 'myoverride')
# same as with add, not a list
assert_equal(cfg['user.name'], 'myoverride')
# but then there is
cfg.add('user.name', 'myother', scope='override')
assert_equal(cfg['user.name'], ['myoverride', 'myother'])
# rename
assert_not_in('ups.name', cfg)
cfg.rename_section('user', 'ups', scope='override')
# original variable still there
assert_in('user.name', cfg)
# rename of override in effect
assert_equal(cfg['ups.name'], ['myoverride', 'myother'])
# remove entirely by section
cfg.remove_section('ups', scope='override')
from datalad.utils import Path
assert_not_in('ups.name', cfg, (
cfg._stores,
cfg.overrides,
))
def test_from_env_overrides():
cfg = ConfigManager()
assert_not_in("datalad.FoO", cfg)
# Some details, like case and underscores, cannot be handled by the direct
# environment variable mapping.
with patch.dict("os.environ", {"DATALAD_FOO": "val"}):
cfg.reload()
assert_not_in("datalad.FoO", cfg)
assert_equal(cfg["datalad.foo"], "val")
# But they can be handled via DATALAD_CONFIG_OVERRIDES_JSON.
with patch.dict(
"os.environ",
{"DATALAD_CONFIG_OVERRIDES_JSON": '{"datalad.FoO": "val"}'}):
cfg.reload()
assert_equal(cfg["datalad.FoO"], "val")
# DATALAD_CONFIG_OVERRIDES_JSON isn't limited to datalad variables.
with patch.dict("os.environ",
{"DATALAD_CONFIG_OVERRIDES_JSON": '{"a.b.c": "val"}'}):
cfg.reload()
assert_equal(cfg["a.b.c"], "val")
# Explicitly provided DATALAD_ variables take precedence over those in
# DATALAD_CONFIG_OVERRIDES_JSON.
with patch.dict(
"os.environ", {
"DATALAD_CONFIG_OVERRIDES_JSON": '{"datalad.foo": "val"}',
"DATALAD_FOO": "val-direct"
}):
cfg.reload()
assert_equal(cfg["datalad.foo"], "val-direct")
# JSON decode errors don't lead to crash.
with patch.dict("os.environ", {"DATALAD_CONFIG_OVERRIDES_JSON": '{'}):
with swallow_logs(logging.WARNING) as cml:
cfg.reload()
assert_in("Failed to load DATALAD_CONFIG_OVERRIDE", cml.out)
def test_from_env():
cfg = ConfigManager()
assert_not_in('datalad.crazy.cfg', cfg)
with patch.dict('os.environ', {'DATALAD_CRAZY_CFG': 'impossibletoguess'}):
cfg.reload()
assert_in('datalad.crazy.cfg', cfg)
assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess')
# not in dataset-only mode
cfg = ConfigManager(Dataset('nowhere'), source='branch')
assert_not_in('datalad.crazy.cfg', cfg)
# check env trumps override
cfg = ConfigManager()
assert_not_in('datalad.crazy.override', cfg)
cfg.set('datalad.crazy.override', 'fromoverride', scope='override')
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromoverride')
with patch.dict('os.environ', {'DATALAD_CRAZY_OVERRIDE': 'fromenv'}):
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_obtain(path=None):
ds = create(path)
cfg = ConfigManager(ds)
dummy = 'datalad.test.dummy'
# we know nothing and we don't know how to ask
assert_raises(RuntimeError, cfg.obtain, dummy)
# can report known ones
cfg.add(dummy, '5.3')
assert_equal(cfg.obtain(dummy), '5.3')
# better type
assert_equal(cfg.obtain(dummy, valtype=float), 5.3)
# don't hide type issues, float doesn't become an int magically
assert_raises(ValueError, cfg.obtain, dummy, valtype=int)
# inject some prior knowledge
from datalad.interface.common_cfg import definitions as cfg_defs
cfg_defs[dummy] = dict(type=float)
# no we don't need to specify a type anymore
assert_equal(cfg.obtain(dummy), 5.3)
# but if we remove the value from the config, all magic is gone
cfg.unset(dummy)
# we know nothing and we don't know how to ask
assert_raises(RuntimeError, cfg.obtain, dummy)
#
# test actual interaction
#
@with_testsui()
def ask():
# fail on unknown dialog type
assert_raises(ValueError,
cfg.obtain,
dummy,
dialog_type='Rorschach_test')
ask()
# ask nicely, and get a value of proper type using the preconfiguration
@with_testsui(responses='5.3')
def ask():
assert_equal(cfg.obtain(dummy, dialog_type='question', text='Tell me'),
5.3)
ask()
# preconfigure even more, to get the most compact call
cfg_defs[dummy]['ui'] = ('question',
dict(text='tell me', title='Gretchen Frage'))
@with_testsui(responses='5.3')
def ask():
assert_equal(cfg.obtain(dummy), 5.3)
ask()
@with_testsui(responses='murks')
def ask():
assert_raises(ValueError, cfg.obtain, dummy)
ask()
# fail to store when destination is not specified, will not even ask
@with_testsui()
def ask():
assert_raises(ValueError, cfg.obtain, dummy, store=True)
ask()
# but we can preconfigure it
cfg_defs[dummy]['destination'] = 'broken'
@with_testsui(responses='5.3')
def ask():
assert_raises(ValueError, cfg.obtain, dummy, store=True)
ask()
# fixup destination
cfg_defs[dummy]['destination'] = 'branch'
@with_testsui(responses='5.3')
def ask():
assert_equal(cfg.obtain(dummy, store=True), 5.3)
ask()
# now it won't have to ask again
@with_testsui()
def ask():
assert_equal(cfg.obtain(dummy), 5.3)
ask()
# wipe it out again
cfg.unset(dummy)
assert_not_in(dummy, cfg)
def test_something(path=None, new_home=None):
# will refuse to work on dataset without a dataset
assert_raises(ValueError, ConfigManager, source='branch')
# now read the example config
cfg = ConfigManager(GitRepo(opj(path, 'ds'), create=True), source='branch')
assert_equal(len(cfg), 5)
assert_in('something.user', cfg)
# multi-value
assert_equal(len(cfg['something.user']), 2)
assert_equal(cfg['something.user'],
('name=Jane Doe', '[email protected]'))
assert_true(cfg.has_section('something'))
assert_false(cfg.has_section('somethingelse'))
assert_equal(sorted(cfg.sections()),
[u'onemore.complicated の beast with.dot', 'something'])
assert_true(cfg.has_option('something', 'user'))
assert_false(cfg.has_option('something', 'us?er'))
assert_false(cfg.has_option('some?thing', 'user'))
assert_equal(sorted(cfg.options('something')),
['empty', 'myint', 'novalue', 'user'])
assert_equal(cfg.options(u'onemore.complicated の beast with.dot'),
['findme'])
assert_equal(sorted(cfg.items()),
[(u'onemore.complicated の beast with.dot.findme', '5.0'),
('something.empty', ''), ('something.myint', '3'),
('something.novalue', None),
('something.user',
('name=Jane Doe', '[email protected]'))])
assert_equal(sorted(cfg.items('something')),
[('something.empty', ''), ('something.myint', '3'),
('something.novalue', None),
('something.user',
('name=Jane Doe', '[email protected]'))])
# by default get last value only
assert_equal(cfg.get('something.user'), '[email protected]')
# but can get all values
assert_equal(cfg.get('something.user', get_all=True),
('name=Jane Doe', '[email protected]'))
assert_raises(KeyError, cfg.__getitem__, 'somedthing.user')
assert_equal(
cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0)
assert_equal(cfg.getint('something', 'myint'), 3)
assert_equal(cfg.getbool('something', 'myint'), True)
# git demands a key without value at all to be used as a flag, thus True
assert_equal(cfg.getbool('something', 'novalue'), True)
assert_equal(cfg.get('something.novalue'), None)
# empty value is False
assert_equal(cfg.getbool('something', 'empty'), False)
assert_equal(cfg.get('something.empty'), '')
assert_equal(cfg.getbool('doesnot', 'exist', default=True), True)
assert_raises(TypeError, cfg.getbool, 'something', 'user')
# gitpython-style access
assert_equal(cfg.get('something.myint'),
cfg.get_value('something', 'myint'))
assert_equal(cfg.get_value('doesnot', 'exist', default='oohaaa'), 'oohaaa')
# weird, but that is how it is
assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None)
# modification follows
cfg.add('something.new', 'の')
assert_equal(cfg.get('something.new'), u'の')
# sections are added on demand
cfg.add('unheard.of', 'fame')
assert_true(cfg.has_section('unheard.of'))
comp = cfg.items('something')
cfg.rename_section('something', 'this')
assert_true(cfg.has_section('this'))
assert_false(cfg.has_section('something'))
# direct comparison would fail, because of section prefix
assert_equal(len(cfg.items('this')), len(comp))
# fail if no such section
with swallow_logs():
assert_raises(CommandError, cfg.rename_section, 'nothere',
'irrelevant')
assert_true(cfg.has_option('this', 'myint'))
cfg.unset('this.myint')
assert_false(cfg.has_option('this', 'myint'))
# batch a changes
cfg.add('mike.wants.to', 'know', reload=False)
assert_false('mike.wants.to' in cfg)
cfg.add('mike.wants.to', 'eat')
assert_true('mike.wants.to' in cfg)
assert_equal(len(cfg['mike.wants.to']), 2)
# set a new one:
cfg.set('mike.should.have', 'known')
assert_in('mike.should.have', cfg)
assert_equal(cfg['mike.should.have'], 'known')
# set an existing one:
cfg.set('mike.should.have', 'known better')
assert_equal(cfg['mike.should.have'], 'known better')
# set, while there are several matching ones already:
cfg.add('mike.should.have', 'a meal')
assert_equal(len(cfg['mike.should.have']), 2)
# raises with force=False
assert_raises(CommandError,
cfg.set,
'mike.should.have',
'a beer',
force=False)
assert_equal(len(cfg['mike.should.have']), 2)
# replaces all matching ones with force=True
cfg.set('mike.should.have', 'a beer', force=True)
assert_equal(cfg['mike.should.have'], 'a beer')
# test deprecated 'where' interface and old 'dataset' (not 'branch') value
# TODO: remove along with the removal of deprecated 'where'
cfg.set('mike.should.have', 'wasknown', where='dataset')
assert_equal(cfg['mike.should.have'], 'wasknown')
assert_equal(cfg.get_from_source('dataset', 'mike.should.have'),
'wasknown')
# fails unknown location
assert_raises(ValueError, cfg.add, 'somesuch', 'shit', scope='umpalumpa')
# very carefully test non-local config
# so carefully that even in case of bad weather Yarik doesn't find some
# lame datalad unittest sections in his precious ~/.gitconfig
# Note: An easier way to test this, would be to just set GIT_CONFIG_GLOBAL
# to point somewhere else. However, this is not supported by git before
# 2.32. Hence, stick with changed HOME in this test, but be sure to unset a
# possible GIT_CONFIG_GLOBAL in addition.
patched_env = os.environ.copy()
patched_env.pop('GIT_CONFIG_GLOBAL', None)
patched_env.update(get_home_envvars(new_home))
with patch.dict('os.environ',
dict(patched_env, DATALAD_SNEAKY_ADDITION='ignore'),
clear=True):
global_gitconfig = opj(new_home, '.gitconfig')
assert (not exists(global_gitconfig))
globalcfg = ConfigManager()
assert_not_in('datalad.unittest.youcan', globalcfg)
assert_in('datalad.sneaky.addition', globalcfg)
cfg.add('datalad.unittest.youcan', 'removeme', scope='global')
assert (exists(global_gitconfig))
# it did not go into the dataset's config!
assert_not_in('datalad.unittest.youcan', cfg)
# does not monitor additions!
globalcfg.reload(force=True)
assert_in('datalad.unittest.youcan', globalcfg)
with swallow_logs():
assert_raises(CommandError,
globalcfg.unset,
'datalad.unittest.youcan',
scope='local')
assert (globalcfg.has_section('datalad.unittest'))
globalcfg.unset('datalad.unittest.youcan', scope='global')
# but after we unset the only value -- that section is no longer listed
assert (not globalcfg.has_section('datalad.unittest'))
assert_not_in('datalad.unittest.youcan', globalcfg)
ok_file_has_content(global_gitconfig, "")
cfg = ConfigManager(Dataset(opj(path, 'ds')),
source='branch',
overrides={'datalad.godgiven': True})
assert_equal(cfg.get('datalad.godgiven'), True)
# setter has no effect
cfg.set('datalad.godgiven', 'false')
assert_equal(cfg.get('datalad.godgiven'), True)
def _install_subds_from_flexible_source(ds, sm, **kwargs):
"""Tries to obtain a given subdataset from several meaningful locations
Parameters
----------
ds : Dataset
Parent dataset of to-be-installed subdataset.
sm : dict
Submodule record as produced by `subdatasets()`.
**kwargs
Passed onto clone()
"""
sm_path = op.relpath(sm['path'], start=sm['parentds'])
# compose a list of candidate clone URLs
clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm)
# prevent inevitable exception from `clone`
dest_path = op.join(ds.path, sm_path)
clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path]
if not clone_urls:
# yield error
yield get_status_dict(
action='install',
ds=ds,
status='error',
message=("Have got no candidates to install subdataset %s from.",
sm_path),
logger=lgr,
)
return
for res in clone_dataset(clone_urls_, Dataset(dest_path), **kwargs):
# make sure to fix a detached HEAD before yielding the install success
# result. The resetting of the branch would undo any change done
# to the repo by processing in response to the result
if res.get('action', None) == 'install' and \
res.get('status', None) == 'ok' and \
res.get('type', None) == 'dataset' and \
res.get('path', None) == dest_path:
_fixup_submodule_dotgit_setup(ds, sm_path)
# do fancy update
lgr.debug(
"Update cloned subdataset {0} in parent".format(dest_path))
ds.repo.update_submodule(sm_path, init=True)
yield res
subds = Dataset(dest_path)
if not subds.is_installed():
lgr.debug('Desired subdataset %s did not materialize, stopping', subds)
return
# check whether clone URL generators were involved
cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)]
if cand_cfg:
# get a handle on the configuration that is specified in the
# dataset itself (local and dataset)
super_cfg = ConfigManager(dataset=ds, source='dataset-local')
need_reload = False
for rec in cand_cfg:
# check whether any of this configuration originated from the
# superdataset. if so, inherit the config in the new subdataset
# clone. if not, keep things clean in order to be able to move with
# any outside configuration change
for c in ('datalad.get.subdataset-source-candidate-{}{}'.format(
rec['cost'], rec['name']),
'datalad.get.subdataset-source-candidate-{}'.format(
rec['name'])):
if c in super_cfg.keys():
subds.config.set(c,
super_cfg.get(c),
where='local',
reload=False)
need_reload = True
break
if need_reload:
subds.config.reload(force=True)
def test_obtain(path):
ds = create(path)
cfg = ConfigManager(ds)
dummy = 'datalad.test.dummy'
# we know nothing and we don't know how to ask
assert_raises(RuntimeError, cfg.obtain, dummy)
# can report known ones
cfg.add(dummy, '5.3')
assert_equal(cfg.obtain(dummy), '5.3')
# better type
assert_equal(cfg.obtain(dummy, valtype=float), 5.3)
# don't hide type issues, float doesn't become an int magically
assert_raises(ValueError, cfg.obtain, dummy, valtype=int)
# inject some prior knowledge
from datalad.interface.common_cfg import definitions as cfg_defs
cfg_defs[dummy] = dict(type=float)
# no we don't need to specify a type anymore
assert_equal(cfg.obtain(dummy), 5.3)
# but if we remove the value from the config, all magic is gone
cfg.unset(dummy)
# we know nothing and we don't know how to ask
assert_raises(RuntimeError, cfg.obtain, dummy)
#
# test actual interaction
#
@with_testsui()
def ask():
# fail on unkown dialog type
assert_raises(ValueError, cfg.obtain, dummy, dialog_type='Rorschach_test')
ask()
# ask nicely, and get a value of proper type using the preconfiguration
@with_testsui(responses='5.3')
def ask():
assert_equal(
cfg.obtain(dummy, dialog_type='question', text='Tell me'), 5.3)
ask()
# preconfigure even more, to get the most compact call
cfg_defs[dummy]['ui'] = ('question', dict(text='tell me', title='Gretchen Frage'))
@with_testsui(responses='5.3')
def ask():
assert_equal(cfg.obtain(dummy), 5.3)
ask()
@with_testsui(responses='murks')
def ask():
assert_raises(ValueError, cfg.obtain, dummy)
ask()
# fail to store when destination is not specified, will not even ask
@with_testsui()
def ask():
assert_raises(ValueError, cfg.obtain, dummy, store=True)
ask()
# but we can preconfigure it
cfg_defs[dummy]['destination'] = 'broken'
@with_testsui(responses='5.3')
def ask():
assert_raises(ValueError, cfg.obtain, dummy, store=True)
ask()
# fixup destination
cfg_defs[dummy]['destination'] = 'dataset'
@with_testsui(responses='5.3')
def ask():
assert_equal(cfg.obtain(dummy, store=True), 5.3)
ask()
# now it won't have to ask again
@with_testsui()
def ask():
assert_equal(cfg.obtain(dummy), 5.3)
ask()
# wipe it out again
cfg.unset(dummy)
assert_not_in(dummy, cfg)
def test_something(path, new_home):
# will refuse to work on dataset without a dataset
assert_raises(ValueError, ConfigManager, source='dataset')
# now read the example config
cfg = ConfigManager(Dataset(opj(path, 'ds')), source='dataset')
assert_equal(len(cfg), 5)
assert_in('something.user', cfg)
# multi-value
assert_equal(len(cfg['something.user']), 2)
assert_equal(cfg['something.user'],
('name=Jane Doe', '[email protected]'))
assert_true(cfg.has_section('something'))
assert_false(cfg.has_section('somethingelse'))
assert_equal(sorted(cfg.sections()),
[u'onemore.complicated の beast with.dot', 'something'])
assert_true(cfg.has_option('something', 'user'))
assert_false(cfg.has_option('something', 'us?er'))
assert_false(cfg.has_option('some?thing', 'user'))
assert_equal(sorted(cfg.options('something')),
['empty', 'myint', 'novalue', 'user'])
assert_equal(cfg.options(u'onemore.complicated の beast with.dot'),
['findme'])
assert_equal(sorted(cfg.items()),
[(u'onemore.complicated の beast with.dot.findme', '5.0'),
('something.empty', ''), ('something.myint', '3'),
('something.novalue', None),
('something.user',
('name=Jane Doe', '[email protected]'))])
assert_equal(sorted(cfg.items('something')),
[('something.empty', ''), ('something.myint', '3'),
('something.novalue', None),
('something.user',
('name=Jane Doe', '[email protected]'))])
# always get all values
assert_equal(cfg.get('something.user'),
('name=Jane Doe', '[email protected]'))
assert_raises(KeyError, cfg.__getitem__, 'somedthing.user')
assert_equal(
cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0)
assert_equal(cfg.getint('something', 'myint'), 3)
assert_equal(cfg.getbool('something', 'myint'), True)
# git demands a key without value at all to be used as a flag, thus True
assert_equal(cfg.getbool('something', 'novalue'), True)
assert_equal(cfg.get('something.novalue'), None)
# empty value is False
assert_equal(cfg.getbool('something', 'empty'), False)
assert_equal(cfg.get('something.empty'), '')
assert_equal(cfg.getbool('doesnot', 'exist', default=True), True)
assert_raises(TypeError, cfg.getbool, 'something', 'user')
# gitpython-style access
assert_equal(cfg.get('something.myint'),
cfg.get_value('something', 'myint'))
assert_equal(cfg.get_value('doesnot', 'exist', default='oohaaa'), 'oohaaa')
# weired, but that is how it is
assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None)
# modification follows
cfg.add('something.new', 'の')
assert_equal(cfg.get('something.new'), u'の')
# sections are added on demand
cfg.add('unheard.of', 'fame')
assert_true(cfg.has_section('unheard.of'))
comp = cfg.items('something')
cfg.rename_section('something', 'this')
assert_true(cfg.has_section('this'))
assert_false(cfg.has_section('something'))
# direct comparision would fail, because of section prefix
assert_equal(len(cfg.items('this')), len(comp))
# fail if no such section
with swallow_logs():
assert_raises(CommandError, cfg.rename_section, 'nothere',
'irrelevant')
assert_true(cfg.has_option('this', 'myint'))
cfg.unset('this.myint')
assert_false(cfg.has_option('this', 'myint'))
# batch a changes
cfg.add('mike.wants.to', 'know', reload=False)
assert_false('mike.wants.to' in cfg)
cfg.add('mike.wants.to', 'eat')
assert_true('mike.wants.to' in cfg)
assert_equal(len(cfg['mike.wants.to']), 2)
# set a new one:
cfg.set('mike.should.have', 'known')
assert_in('mike.should.have', cfg)
assert_equal(cfg['mike.should.have'], 'known')
# set an existing one:
cfg.set('mike.should.have', 'known better')
assert_equal(cfg['mike.should.have'], 'known better')
# set, while there are several matching ones already:
cfg.add('mike.should.have', 'a meal')
assert_equal(len(cfg['mike.should.have']), 2)
# raises with force=False
assert_raises(CommandError,
cfg.set,
'mike.should.have',
'a beer',
force=False)
assert_equal(len(cfg['mike.should.have']), 2)
# replaces all matching ones with force=True
cfg.set('mike.should.have', 'a beer', force=True)
assert_equal(cfg['mike.should.have'], 'a beer')
# fails unknown location
assert_raises(ValueError, cfg.add, 'somesuch', 'shit', where='umpalumpa')
# very carefully test non-local config
# so carefully that even in case of bad weather Yarik doesn't find some
# lame datalad unittest sections in his precious ~/.gitconfig
with patch.dict('os.environ', {
'HOME': new_home,
'DATALAD_SNEAKY_ADDITION': 'ignore'
}):
global_gitconfig = opj(new_home, '.gitconfig')
assert (not exists(global_gitconfig))
globalcfg = ConfigManager()
assert_not_in('datalad.unittest.youcan', globalcfg)
assert_in('datalad.sneaky.addition', globalcfg)
cfg.add('datalad.unittest.youcan', 'removeme', where='global')
assert (exists(global_gitconfig))
# it did not go into the dataset's config!
assert_not_in('datalad.unittest.youcan', cfg)
# does not monitor additions!
globalcfg.reload(force=True)
assert_in('datalad.unittest.youcan', globalcfg)
with swallow_logs():
assert_raises(CommandError,
globalcfg.unset,
'datalad.unittest.youcan',
where='local')
assert (globalcfg.has_section('datalad.unittest'))
globalcfg.unset('datalad.unittest.youcan', where='global')
# but after we unset the only value -- that section is no longer listed
assert (not globalcfg.has_section('datalad.unittest'))
assert_not_in('datalad.unittest.youcan', globalcfg)
if external_versions['cmd:git'] < '2.18':
# older versions leave empty section behind in the file
ok_file_has_content(global_gitconfig,
'[datalad "unittest"]',
strip=True)
# remove_section to clean it up entirely
globalcfg.remove_section('datalad.unittest', where='global')
ok_file_has_content(global_gitconfig, "")
cfg = ConfigManager(Dataset(opj(path, 'ds')),
source='dataset',
overrides={'datalad.godgiven': True})
assert_equal(cfg.get('datalad.godgiven'), True)
# setter has no effect
cfg.set('datalad.godgiven', 'false')
assert_equal(cfg.get('datalad.godgiven'), True)
def test_from_env():
cfg = ConfigManager()
assert_not_in('datalad.crazy.cfg', cfg)
os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess'
cfg.reload()
assert_in('datalad.crazy.cfg', cfg)
assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess')
# not in dataset-only mode
cfg = ConfigManager(Dataset('nowhere'), source='dataset')
assert_not_in('datalad.crazy.cfg', cfg)
# check env trumps override
cfg = ConfigManager()
assert_not_in('datalad.crazy.override', cfg)
cfg.set('datalad.crazy.override', 'fromoverride', where='override')
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromoverride')
os.environ['DATALAD_CRAZY_OVERRIDE'] = 'fromenv'
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_something(path, new_home):
# read nothing, has nothing
cfg = ConfigManager(dataset_only=True)
assert_false(len(cfg))
# now read the example config
cfg = ConfigManager(Dataset(opj(path, 'ds')), dataset_only=True)
assert_equal(len(cfg), 3)
assert_in('something.user', cfg)
# multi-value
assert_equal(len(cfg['something.user']), 2)
assert_equal(cfg['something.user'], ('name=Jane Doe', '[email protected]'))
assert_true(cfg.has_section('something'))
assert_false(cfg.has_section('somethingelse'))
assert_equal(sorted(cfg.sections()),
[u'onemore.complicated の beast with.dot', 'something'])
assert_true(cfg.has_option('something', 'user'))
assert_false(cfg.has_option('something', 'us?er'))
assert_false(cfg.has_option('some?thing', 'user'))
assert_equal(sorted(cfg.options('something')), ['myint', 'user'])
assert_equal(cfg.options(u'onemore.complicated の beast with.dot'), ['findme'])
assert_equal(
sorted(cfg.items()),
[(u'onemore.complicated の beast with.dot.findme', '5.0'),
('something.myint', '3'),
('something.user', ('name=Jane Doe', '[email protected]'))])
assert_equal(
sorted(cfg.items('something')),
[('something.myint', '3'),
('something.user', ('name=Jane Doe', '[email protected]'))])
# always get all values
assert_equal(
cfg.get('something.user'),
('name=Jane Doe', '[email protected]'))
assert_raises(KeyError, cfg.__getitem__, 'somedthing.user')
assert_equal(cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0)
assert_equal(cfg.getint('something', 'myint'), 3)
assert_equal(cfg.getbool('something', 'myint'), True)
assert_equal(cfg.getbool('doesnot', 'exist', default=True), True)
assert_raises(TypeError, cfg.getbool, 'something', 'user')
# gitpython-style access
assert_equal(cfg.get('something.myint'), cfg.get_value('something', 'myint'))
assert_equal(cfg.get_value('doesnot', 'exist', default='oohaaa'), 'oohaaa')
# weired, but that is how it is
assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None)
# modification follows
cfg.add('something.new', 'の')
assert_equal(cfg.get('something.new'), u'の')
# sections are added on demand
cfg.add('unheard.of', 'fame')
assert_true(cfg.has_section('unheard.of'))
comp = cfg.items('something')
cfg.rename_section('something', 'this')
assert_true(cfg.has_section('this'))
assert_false(cfg.has_section('something'))
# direct comparision would fail, because of section prefix
assert_equal(len(cfg.items('this')), len(comp))
# fail if no such section
with swallow_logs():
assert_raises(CommandError, cfg.rename_section, 'nothere', 'irrelevant')
assert_true(cfg.has_option('this', 'myint'))
cfg.unset('this.myint')
assert_false(cfg.has_option('this', 'myint'))
# batch a changes
cfg.add('mike.wants.to', 'know', reload=False)
assert_false('mike.wants.to' in cfg)
cfg.add('mike.wants.to', 'eat')
assert_true('mike.wants.to' in cfg)
assert_equal(len(cfg['mike.wants.to']), 2)
# set a new one:
cfg.set('mike.should.have', 'known')
assert_in('mike.should.have', cfg)
assert_equal(cfg['mike.should.have'], 'known')
# set an existing one:
cfg.set('mike.should.have', 'known better')
assert_equal(cfg['mike.should.have'], 'known better')
# set, while there are several matching ones already:
cfg.add('mike.should.have', 'a meal')
assert_equal(len(cfg['mike.should.have']), 2)
# raises with force=False
assert_raises(CommandError,
cfg.set, 'mike.should.have', 'a beer', force=False)
assert_equal(len(cfg['mike.should.have']), 2)
# replaces all matching ones with force=True
cfg.set('mike.should.have', 'a beer', force=True)
assert_equal(cfg['mike.should.have'], 'a beer')
# fails unknown location
assert_raises(ValueError, cfg.add, 'somesuch', 'shit', where='umpalumpa')
# very carefully test non-local config
# so carefully that even in case of bad weather Yarik doesn't find some
# lame datalad unittest sections in his precious ~/.gitconfig
with patch.dict('os.environ',
{'HOME': new_home, 'DATALAD_SNEAKY_ADDITION': 'ignore'}):
global_gitconfig = opj(new_home, '.gitconfig')
assert(not exists(global_gitconfig))
globalcfg = ConfigManager(dataset_only=False)
assert_not_in('datalad.unittest.youcan', globalcfg)
assert_in('datalad.sneaky.addition', globalcfg)
cfg.add('datalad.unittest.youcan', 'removeme', where='global')
assert(exists(global_gitconfig))
# it did not go into the dataset's config!
assert_not_in('datalad.unittest.youcan', cfg)
# does not monitor additions!
globalcfg.reload(force=True)
assert_in('datalad.unittest.youcan', globalcfg)
with swallow_logs():
assert_raises(
CommandError,
globalcfg.unset,
'datalad.unittest.youcan',
where='local')
assert(globalcfg.has_section('datalad.unittest'))
globalcfg.unset('datalad.unittest.youcan', where='global')
# but after we unset the only value -- that section is no longer listed
assert (not globalcfg.has_section('datalad.unittest'))
assert_not_in('datalad.unittest.youcan', globalcfg)
if external_versions['cmd:git'] < '2.18':
# older versions leave empty section behind in the file
ok_file_has_content(global_gitconfig, '[datalad "unittest"]', strip=True)
# remove_section to clean it up entirely
globalcfg.remove_section('datalad.unittest', where='global')
ok_file_has_content(global_gitconfig, "")
cfg = ConfigManager(
Dataset(opj(path, 'ds')),
dataset_only=True,
overrides={'datalad.godgiven': True})
assert_equal(cfg.get('datalad.godgiven'), True)
# setter has no effect
cfg.set('datalad.godgiven', 'false')
assert_equal(cfg.get('datalad.godgiven'), True)
def eval_func(wrapped, instance, args, kwargs):
lgr.log(2, "Entered eval_func for %s", func)
# for result filters
# we need to produce a dict with argname/argvalue pairs for all args
# incl. defaults and args given as positionals
allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs)
# determine the command class associated with `wrapped`
wrapped_class = get_wrapped_class(wrapped)
# retrieve common options from kwargs, and fall back on the command
# class attributes, or general defaults if needed
kwargs = kwargs.copy() # we will pop, which might cause side-effect
common_params = {
p_name: kwargs.pop(
# go with any explicitly given default
p_name,
# otherwise determine the command class and pull any
# default set in that class
getattr(
wrapped_class,
p_name,
# or the common default
eval_defaults[p_name]))
for p_name in eval_params
}
# short cuts and configured setup for common options
return_type = common_params['return_type']
result_filter = get_result_filter(common_params['result_filter'])
# resolve string labels for transformers too
result_xfm = known_result_xfms.get(
common_params['result_xfm'],
# use verbatim, if not a known label
common_params['result_xfm'])
result_renderer = common_params['result_renderer']
# TODO remove this conditional branch entirely, done outside
if not result_renderer:
result_renderer = dlcfg.get('datalad.api.result-renderer', None)
# look for potential override of logging behavior
result_log_level = dlcfg.get('datalad.log.result-level', None)
# query cfg for defaults
# .is_installed and .config can be costly, so ensure we do
# it only once. See https://github.com/datalad/datalad/issues/3575
dataset_arg = allkwargs.get('dataset', None)
from datalad.distribution.dataset import Dataset
ds = dataset_arg if isinstance(dataset_arg, Dataset) \
else Dataset(dataset_arg) if dataset_arg else None
# do not reuse a dataset's existing config manager here
# they are configured to read the committed dataset configuration
# too. That means a datalad update can silently bring in new
# procedure definitions from the outside, and in some sense enable
# remote code execution by a 3rd-party
# To avoid that, create a new config manager that only reads local
# config (system and .git/config), plus any overrides given to this
# datalad session
proc_cfg = ConfigManager(
ds, source='local',
overrides=dlcfg.overrides) if ds and ds.is_installed() else dlcfg
# look for hooks
hooks = get_jsonhooks_from_config(proc_cfg)
# this internal helper function actually drives the command
# generator-style, it may generate an exception if desired,
# on incomplete results
def generator_func(*_args, **_kwargs):
# flag whether to raise an exception
incomplete_results = []
# track what actions were performed how many times
action_summary = {}
# if a custom summary is to be provided, collect the results
# of the command execution
results = []
do_custom_result_summary = result_renderer in ('tailored', 'default') \
and hasattr(wrapped_class, 'custom_result_summary_renderer')
# process main results
for r in _process_results(
# execution
wrapped(*_args, **_kwargs),
wrapped_class,
common_params['on_failure'],
# bookkeeping
action_summary,
incomplete_results,
# communication
result_renderer,
result_log_level,
# let renderers get to see how a command was called
allkwargs):
for hook, spec in hooks.items():
# run the hooks before we yield the result
# this ensures that they are executed before
# a potentially wrapper command gets to act
# on them
if match_jsonhook2result(hook, r, spec['match']):
lgr.debug('Result %s matches hook %s', r, hook)
# a hook is also a command that yields results
# so yield them outside too
# users need to pay attention to void infinite
# loops, i.e. when a hook yields a result that
# triggers that same hook again
for hr in run_jsonhook(hook, spec, r, dataset_arg):
# apply same logic as for main results, otherwise
# any filters would only tackle the primary results
# and a mixture of return values could happen
if not keep_result(hr, result_filter, **allkwargs):
continue
hr = xfm_result(hr, result_xfm)
# rationale for conditional is a few lines down
if hr:
yield hr
if not keep_result(r, result_filter, **allkwargs):
continue
r = xfm_result(r, result_xfm)
# in case the result_xfm decided to not give us anything
# exclude it from the results. There is no particular reason
# to do so other than that it was established behavior when
# this comment was written. This will not affect any real
# result record
if r:
yield r
# collect if summary is desired
if do_custom_result_summary:
results.append(r)
# result summary before a potential exception
# custom first
if do_custom_result_summary:
wrapped_class.custom_result_summary_renderer(results)
elif result_renderer == 'default' and action_summary and \
sum(sum(s.values()) for s in action_summary.values()) > 1:
# give a summary in default mode, when there was more than one
# action performed
ui.message("action summary:\n {}".format('\n '.join(
'{} ({})'.format(
act, ', '.join(
'{}: {}'.format(status, action_summary[act]
[status])
for status in sorted(action_summary[act])))
for act in sorted(action_summary))))
if incomplete_results:
raise IncompleteResultsError(
failed=incomplete_results,
msg="Command did not complete successfully")
if return_type == 'generator':
# hand over the generator
lgr.log(2, "Returning generator_func from eval_func for %s",
wrapped_class)
return generator_func(*args, **kwargs)
else:
@wrapt.decorator
def return_func(wrapped_, instance_, args_, kwargs_):
results = wrapped_(*args_, **kwargs_)
if inspect.isgenerator(results):
# unwind generator if there is one, this actually runs
# any processing
results = list(results)
# render summaries
if not result_xfm and result_renderer in ('tailored',
'default'):
# cannot render transformed results
if hasattr(wrapped_class,
'custom_result_summary_renderer'):
wrapped_class.custom_result_summary_renderer(results)
if return_type == 'item-or-list' and \
len(results) < 2:
return results[0] if results else None
else:
return results
lgr.log(2, "Returning return_func from eval_func for %s",
wrapped_class)
return return_func(generator_func)(*args, **kwargs)
def _install_subds_from_flexible_source(ds, sm, **kwargs):
"""Tries to obtain a given subdataset from several meaningful locations
Parameters
----------
ds : Dataset
Parent dataset of to-be-installed subdataset.
sm : dict
Submodule record as produced by `subdatasets()`.
**kwargs
Passed onto clone()
"""
sm_path = op.relpath(sm['path'], start=sm['parentds'])
# compose a list of candidate clone URLs
clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm)
# prevent inevitable exception from `clone`
dest_path = op.join(ds.path, sm_path)
clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path]
if not clone_urls:
# yield error
yield get_status_dict(
action='install',
ds=ds,
status='error',
message=("Have got no candidates to install subdataset %s from.",
sm_path),
logger=lgr,
)
return
for res in clone_dataset(clone_urls_,
Dataset(dest_path),
cfg=ds.config,
checkout_gitsha=sm['gitshasum'],
**kwargs):
if res.get('action', None) == 'install' and \
res.get('status', None) == 'ok' and \
res.get('type', None) == 'dataset' and \
res.get('path', None) == dest_path:
_fixup_submodule_dotgit_setup(ds, sm_path)
section_name = 'submodule.{}'.format(sm['gitmodule_name'])
# register the submodule as "active" in the superdataset
ds.config.set(
'{}.active'.format(section_name),
'true',
reload=False,
force=True,
where='local',
)
ds.config.set(
'{}.url'.format(section_name),
# record the actual source URL of the successful clone
# and not a funky prediction based on the parent ds
# like ds.repo.update_submodule() would do (does not
# accept a URL)
res['source']['giturl'],
reload=True,
force=True,
where='local',
)
yield res
subds = Dataset(dest_path)
if not subds.is_installed():
lgr.debug('Desired subdataset %s did not materialize, stopping', subds)
return
# check whether clone URL generators were involved
cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)]
if cand_cfg:
# get a handle on the configuration that is specified in the
# dataset itself (local and dataset)
super_cfg = ConfigManager(dataset=ds, source='dataset-local')
need_reload = False
for rec in cand_cfg:
# check whether any of this configuration originated from the
# superdataset. if so, inherit the config in the new subdataset
# clone. if not, keep things clean in order to be able to move with
# any outside configuration change
for c in ('datalad.get.subdataset-source-candidate-{}{}'.format(
rec['cost'], rec['name']),
'datalad.get.subdataset-source-candidate-{}'.format(
rec['name'])):
if c in super_cfg.keys():
subds.config.set(c,
super_cfg.get(c),
where='local',
reload=False)
need_reload = True
break
if need_reload:
subds.config.reload(force=True)
def __init__(self, dataset, name=None, email=None, where='local'):
self.config_manager = ConfigManager(dataset)
self.email = email if email else SERVICE_EMAIL
self.name = name if name else SERVICE_USER
self.where = where
def test_crazy_cfg(path):
cfg = ConfigManager(Dataset(opj(path, 'ds')), dataset_only=True)
assert_in('crazy.padry', cfg)
def _install_subds_from_flexible_source(ds, sm, **kwargs):
"""Tries to obtain a given subdataset from several meaningful locations
Parameters
----------
ds : Dataset
Parent dataset of to-be-installed subdataset.
sm : dict
Submodule record as produced by `subdatasets()`.
**kwargs
Passed onto clone()
"""
sm_path = op.relpath(sm['path'], start=sm['parentds'])
# compose a list of candidate clone URLs
clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm)
# prevent inevitable exception from `clone`
dest_path = op.join(ds.path, sm_path)
clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path]
if not clone_urls:
# yield error
yield get_status_dict(
action='install',
ds=ds,
status='error',
message=("Have got no candidates to install subdataset %s from.",
sm_path),
logger=lgr,
)
return
for res in clone_dataset(clone_urls_,
Dataset(dest_path),
cfg=ds.config,
**kwargs):
# make sure to fix a detached HEAD before yielding the install success
# result. The resetting of the branch would undo any change done
# to the repo by processing in response to the result
if res.get('action', None) == 'install' and \
res.get('status', None) == 'ok' and \
res.get('type', None) == 'dataset' and \
res.get('path', None) == dest_path:
_fixup_submodule_dotgit_setup(ds, sm_path)
target_commit = sm['gitshasum']
lgr.debug(
"Update cloned subdataset {0} in parent".format(dest_path))
section_name = 'submodule.{}'.format(sm['gitmodule_name'])
# do not use `git-submodule update --init`, it would make calls
# to git-config which will not obey datalad inter-process locks for
# modifying .git/config
sub = GitRepo(res['path'])
# record what branch we were on right after the clone
# TODO instead of the active branch, this should first consider
# a configured branch in the submodule record of the superdataset
sub_orig_branch = sub.get_active_branch()
# if we are on a branch this hexsha will be the tip of that branch
sub_orig_hexsha = sub.get_hexsha()
if sub_orig_hexsha != target_commit:
# make sure we have the desired commit locally
# expensive and possibly error-prone fetch conditional on cheap
# local check
if not sub.commit_exists(target_commit):
try:
sub.fetch(remote='origin', refspec=target_commit)
except CommandError:
pass
# instead of inspecting the fetch results for possible ways
# with which it could failed to produced the desired result
# let's verify the presence of the commit directly, we are in
# expensive-land already anyways
if not sub.commit_exists(target_commit):
res.update(
status='error',
message=
('Target commit %s does not exist in the clone, and '
'a fetch that commit from origin failed',
target_commit[:8]),
)
yield res
# there is nothing we can do about this
# MIH thinks that removing the clone is not needed, as a likely
# next step will have to be a manual recovery intervention
# and not another blind attempt
continue
# checkout the desired commit
sub.call_git(['checkout', target_commit])
# did we detach?
# XXX: This is a less generic variant of a part of
# GitRepo.update_submodule(). It makes use of already available
# information and trusts the existence of the just cloned repo
# and avoids (redoing) some safety checks
if sub_orig_branch and not sub.get_active_branch():
# trace if current state is a predecessor of the branch_hexsha
lgr.debug(
"Detached HEAD after updating submodule %s "
"(original branch: %s)", sub, sub_orig_branch)
if sub.get_merge_base([sub_orig_hexsha,
target_commit]) == target_commit:
# TODO: config option?
# MIH: There is no real need here. IMHO this should all not
# happen, unless the submodule record has a branch
# configured. And Datalad should leave such a record, when
# a submodule is registered.
# we assume the target_commit to be from the same branch,
# because it is an ancestor -- update that original branch
# to point to the target_commit, and update HEAD to point to
# that location -- this readies the subdataset for
# further modification
lgr.info(
"Reset subdataset branch '%s' to %s (from %s) to "
"avoid a detached HEAD", sub_orig_branch,
target_commit[:8], sub_orig_hexsha[:8])
branch_ref = 'refs/heads/%s' % sub_orig_branch
sub.update_ref(branch_ref, target_commit)
sub.update_ref('HEAD', branch_ref, symbolic=True)
else:
lgr.warning(
"%s has a detached HEAD, because the recorded "
"subdataset state %s has no unique ancestor with "
"branch '%s'", sub, target_commit[:8],
sub_orig_branch)
# register the submodule as "active" in the superdataset
ds.config.set(
'{}.active'.format(section_name),
'true',
reload=False,
force=True,
where='local',
)
ds.config.set(
'{}.url'.format(section_name),
# record the actual source URL of the successful clone
# and not a funky prediction based on the parent ds
# like ds.repo.update_submodule() would do (does not
# accept a URL)
res['source']['giturl'],
reload=True,
force=True,
where='local',
)
yield res
subds = Dataset(dest_path)
if not subds.is_installed():
lgr.debug('Desired subdataset %s did not materialize, stopping', subds)
return
# check whether clone URL generators were involved
cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)]
if cand_cfg:
# get a handle on the configuration that is specified in the
# dataset itself (local and dataset)
super_cfg = ConfigManager(dataset=ds, source='dataset-local')
need_reload = False
for rec in cand_cfg:
# check whether any of this configuration originated from the
# superdataset. if so, inherit the config in the new subdataset
# clone. if not, keep things clean in order to be able to move with
# any outside configuration change
for c in ('datalad.get.subdataset-source-candidate-{}{}'.format(
rec['cost'], rec['name']),
'datalad.get.subdataset-source-candidate-{}'.format(
rec['name'])):
if c in super_cfg.keys():
subds.config.set(c,
super_cfg.get(c),
where='local',
reload=False)
need_reload = True
break
if need_reload:
subds.config.reload(force=True)
