def test_from_env(): cfg = ConfigManager() assert_not_in('datalad.crazy.cfg', cfg) os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess' cfg.reload() assert_in('datalad.crazy.cfg', cfg) assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess') # not in dataset-only mode cfg = ConfigManager(Dataset('nowhere'), dataset_only=True) assert_not_in('datalad.crazy.cfg', cfg)
def test_crazy_cfg(path): cfg = ConfigManager(Dataset(opj(path, 'ds')), source='dataset') assert_in('crazy.padry', cfg) # make sure crazy config is not read when in local mode cfg = ConfigManager(Dataset(opj(path, 'ds')), source='local') assert_not_in('crazy.padry', cfg) # it will make it in in 'any' mode though cfg = ConfigManager(Dataset(opj(path, 'ds')), source='any') assert_in('crazy.padry', cfg) # typos in the source mode arg will not have silent side-effects assert_raises( ValueError, ConfigManager, Dataset(opj(path, 'ds')), source='locale')
def test_inherit_src_candidates(lcl, storepath, url): lcl = Path(lcl) storepath = Path(storepath) # dataset with a subdataset ds1 = Dataset(lcl / 'ds1').create() ds1sub = ds1.create('sub') # a different dataset into which we install ds1, but do not touch its subds ds2 = Dataset(lcl / 'ds2').create() ds2.clone(source=ds1.path, path='mysub') # we give no dataset a source candidate config! # move all dataset into the store for d in (ds1, ds1sub, ds2): _move2store(storepath, d) # now we must be able to obtain all three datasets from the store riaclone = clone( 'ria+{}#{}'.format( # store URL url, # ID of the root dataset ds2.id), lcl / 'clone', ) # what happens is the the initial clone call sets a source candidate # config, because it sees the dataset coming from a store # all obtained subdatasets get the config inherited on-clone datasets = riaclone.get('.', get_data=False, recursive=True, result_xfm='datasets') # we get two subdatasets eq_(len(datasets), 2) for ds in datasets: eq_(ConfigManager(dataset=ds, source='dataset-local').get( 'datalad.get.subdataset-source-candidate-200origin'), 'ria+%s#{id}' % url)
def config(self): """Get an instance of the parser for the persistent dataset configuration. Note, that this property is evaluated every time it is used. If used multiple times within a function it's probably a good idea to store its value in a local variable and use this variable instead. Returns ------- ConfigManager """ if self.repo is None: # if there's no repo (yet or anymore), we can't read/write config at # dataset level, but only at user/system level # However, if this was the case before as well, we don't want a new # instance of ConfigManager if self._cfg_bound in (True, None): self._cfg = ConfigManager(dataset=None, dataset_only=False) self._cfg_bound = False else: self._cfg = self.repo.config self._cfg_bound = True return self._cfg
def config(self): """Get an instance of the parser for the persistent dataset configuration. Returns ------- ConfigManager """ if self._cfg is None: # associate with this dataset and read the entire config hierarchy self._cfg = ConfigManager(dataset=self, dataset_only=False) return self._cfg
def cfg(self): """Get a ConfigManager instance for this repository Returns ------- ConfigManager """ if self._cfg is None: # associate with this dataset and read the entire config hierarchy self._cfg = ConfigManager(dataset=self, source='any') return self._cfg
def test_dataset_local_mode(path=None): ds = create(path) # any sensible (and also our CI) test environment(s) should have this assert_in('user.name', ds.config) # from .datalad/config assert_in('datalad.dataset.id', ds.config) # from .git/config assert_in('annex.version', ds.config) # now check that dataset-local mode doesn't have the global piece cfg = ConfigManager(ds, source='branch-local') assert_not_in('user.name', cfg) assert_in('datalad.dataset.id', cfg) assert_in('annex.version', cfg)
class CommitInfo(): """Context manager for setting commit info on datalad operations that use it.""" def __init__(self, dataset, name=None, email=None, where='local'): self.config_manager = ConfigManager(dataset) self.email = email if email else SERVICE_EMAIL self.name = name if name else SERVICE_USER self.where = where def __enter__(self): self.config_manager.set('user.email', self.email, self.where) self.config_manager.set('user.name', self.name, self.where) def __exit__(self, exception_type, exception_value, traceback): self.config_manager.set('user.email', SERVICE_EMAIL, self.where) self.config_manager.set('user.name', SERVICE_USER, self.where)
def test_from_env(): cfg = ConfigManager() assert_not_in('datalad.crazy.cfg', cfg) os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess' cfg.reload() assert_in('datalad.crazy.cfg', cfg) assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess') # not in dataset-only mode cfg = ConfigManager(Dataset('nowhere'), dataset_only=True) assert_not_in('datalad.crazy.cfg', cfg) # check env trumps override cfg = ConfigManager() assert_not_in('datalad.crazy.override', cfg) cfg.overrides['datalad.crazy.override'] = 'fromoverride' cfg.reload() assert_equal(cfg['datalad.crazy.override'], 'fromoverride') os.environ['DATALAD_CRAZY_OVERRIDE'] = 'fromenv' cfg.reload() assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_dataset_systemglobal_mode(path=None): ds = create(path) # any sensible (and also our CI) test environment(s) should have this assert_in('user.name', ds.config) # from .datalad/config assert_in('datalad.dataset.id', ds.config) # from .git/config assert_in('annex.version', ds.config) with chpwd(path): # now check that no config from a random dataset at PWD is picked up # if not dataset instance was provided cfg = ConfigManager(dataset=None, source='any') assert_in('user.name', cfg) assert_not_in('datalad.dataset.id', cfg) assert_not_in('annex.version', cfg)
def test_no_local_write_if_no_dataset(path=None): Dataset(path).create() with chpwd(path): cfg = ConfigManager() with assert_raises(CommandError): cfg.set('a.b.c', 'd', scope='local')
def test_overrides(): cfg = ConfigManager() # any sensible (and also our CI) test environment(s) should have this assert_in('user.name', cfg) # set cfg.set('user.name', 'myoverride', scope='override') assert_equal(cfg['user.name'], 'myoverride') # unset just removes override, not entire config cfg.unset('user.name', scope='override') assert_in('user.name', cfg) assert_not_equal('user.name', 'myoverride') # add # there is no initial increment cfg.add('user.name', 'myoverride', scope='override') assert_equal(cfg['user.name'], 'myoverride') # same as with add, not a list assert_equal(cfg['user.name'], 'myoverride') # but then there is cfg.add('user.name', 'myother', scope='override') assert_equal(cfg['user.name'], ['myoverride', 'myother']) # rename assert_not_in('ups.name', cfg) cfg.rename_section('user', 'ups', scope='override') # original variable still there assert_in('user.name', cfg) # rename of override in effect assert_equal(cfg['ups.name'], ['myoverride', 'myother']) # remove entirely by section cfg.remove_section('ups', scope='override') from datalad.utils import Path assert_not_in('ups.name', cfg, ( cfg._stores, cfg.overrides, ))
def test_from_env_overrides(): cfg = ConfigManager() assert_not_in("datalad.FoO", cfg) # Some details, like case and underscores, cannot be handled by the direct # environment variable mapping. with patch.dict("os.environ", {"DATALAD_FOO": "val"}): cfg.reload() assert_not_in("datalad.FoO", cfg) assert_equal(cfg["datalad.foo"], "val") # But they can be handled via DATALAD_CONFIG_OVERRIDES_JSON. with patch.dict( "os.environ", {"DATALAD_CONFIG_OVERRIDES_JSON": '{"datalad.FoO": "val"}'}): cfg.reload() assert_equal(cfg["datalad.FoO"], "val") # DATALAD_CONFIG_OVERRIDES_JSON isn't limited to datalad variables. with patch.dict("os.environ", {"DATALAD_CONFIG_OVERRIDES_JSON": '{"a.b.c": "val"}'}): cfg.reload() assert_equal(cfg["a.b.c"], "val") # Explicitly provided DATALAD_ variables take precedence over those in # DATALAD_CONFIG_OVERRIDES_JSON. with patch.dict( "os.environ", { "DATALAD_CONFIG_OVERRIDES_JSON": '{"datalad.foo": "val"}', "DATALAD_FOO": "val-direct" }): cfg.reload() assert_equal(cfg["datalad.foo"], "val-direct") # JSON decode errors don't lead to crash. with patch.dict("os.environ", {"DATALAD_CONFIG_OVERRIDES_JSON": '{'}): with swallow_logs(logging.WARNING) as cml: cfg.reload() assert_in("Failed to load DATALAD_CONFIG_OVERRIDE", cml.out)
def test_from_env(): cfg = ConfigManager() assert_not_in('datalad.crazy.cfg', cfg) with patch.dict('os.environ', {'DATALAD_CRAZY_CFG': 'impossibletoguess'}): cfg.reload() assert_in('datalad.crazy.cfg', cfg) assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess') # not in dataset-only mode cfg = ConfigManager(Dataset('nowhere'), source='branch') assert_not_in('datalad.crazy.cfg', cfg) # check env trumps override cfg = ConfigManager() assert_not_in('datalad.crazy.override', cfg) cfg.set('datalad.crazy.override', 'fromoverride', scope='override') cfg.reload() assert_equal(cfg['datalad.crazy.override'], 'fromoverride') with patch.dict('os.environ', {'DATALAD_CRAZY_OVERRIDE': 'fromenv'}): cfg.reload() assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_obtain(path=None): ds = create(path) cfg = ConfigManager(ds) dummy = 'datalad.test.dummy' # we know nothing and we don't know how to ask assert_raises(RuntimeError, cfg.obtain, dummy) # can report known ones cfg.add(dummy, '5.3') assert_equal(cfg.obtain(dummy), '5.3') # better type assert_equal(cfg.obtain(dummy, valtype=float), 5.3) # don't hide type issues, float doesn't become an int magically assert_raises(ValueError, cfg.obtain, dummy, valtype=int) # inject some prior knowledge from datalad.interface.common_cfg import definitions as cfg_defs cfg_defs[dummy] = dict(type=float) # no we don't need to specify a type anymore assert_equal(cfg.obtain(dummy), 5.3) # but if we remove the value from the config, all magic is gone cfg.unset(dummy) # we know nothing and we don't know how to ask assert_raises(RuntimeError, cfg.obtain, dummy) # # test actual interaction # @with_testsui() def ask(): # fail on unknown dialog type assert_raises(ValueError, cfg.obtain, dummy, dialog_type='Rorschach_test') ask() # ask nicely, and get a value of proper type using the preconfiguration @with_testsui(responses='5.3') def ask(): assert_equal(cfg.obtain(dummy, dialog_type='question', text='Tell me'), 5.3) ask() # preconfigure even more, to get the most compact call cfg_defs[dummy]['ui'] = ('question', dict(text='tell me', title='Gretchen Frage')) @with_testsui(responses='5.3') def ask(): assert_equal(cfg.obtain(dummy), 5.3) ask() @with_testsui(responses='murks') def ask(): assert_raises(ValueError, cfg.obtain, dummy) ask() # fail to store when destination is not specified, will not even ask @with_testsui() def ask(): assert_raises(ValueError, cfg.obtain, dummy, store=True) ask() # but we can preconfigure it cfg_defs[dummy]['destination'] = 'broken' @with_testsui(responses='5.3') def ask(): assert_raises(ValueError, cfg.obtain, dummy, store=True) ask() # fixup destination cfg_defs[dummy]['destination'] = 'branch' @with_testsui(responses='5.3') def ask(): assert_equal(cfg.obtain(dummy, store=True), 5.3) ask() # now it won't have to ask again @with_testsui() def ask(): assert_equal(cfg.obtain(dummy), 5.3) ask() # wipe it out again cfg.unset(dummy) assert_not_in(dummy, cfg)
def test_something(path=None, new_home=None): # will refuse to work on dataset without a dataset assert_raises(ValueError, ConfigManager, source='branch') # now read the example config cfg = ConfigManager(GitRepo(opj(path, 'ds'), create=True), source='branch') assert_equal(len(cfg), 5) assert_in('something.user', cfg) # multi-value assert_equal(len(cfg['something.user']), 2) assert_equal(cfg['something.user'], ('name=Jane Doe', '[email protected]')) assert_true(cfg.has_section('something')) assert_false(cfg.has_section('somethingelse')) assert_equal(sorted(cfg.sections()), [u'onemore.complicated の beast with.dot', 'something']) assert_true(cfg.has_option('something', 'user')) assert_false(cfg.has_option('something', 'us?er')) assert_false(cfg.has_option('some?thing', 'user')) assert_equal(sorted(cfg.options('something')), ['empty', 'myint', 'novalue', 'user']) assert_equal(cfg.options(u'onemore.complicated の beast with.dot'), ['findme']) assert_equal(sorted(cfg.items()), [(u'onemore.complicated の beast with.dot.findme', '5.0'), ('something.empty', ''), ('something.myint', '3'), ('something.novalue', None), ('something.user', ('name=Jane Doe', '[email protected]'))]) assert_equal(sorted(cfg.items('something')), [('something.empty', ''), ('something.myint', '3'), ('something.novalue', None), ('something.user', ('name=Jane Doe', '[email protected]'))]) # by default get last value only assert_equal(cfg.get('something.user'), '[email protected]') # but can get all values assert_equal(cfg.get('something.user', get_all=True), ('name=Jane Doe', '[email protected]')) assert_raises(KeyError, cfg.__getitem__, 'somedthing.user') assert_equal( cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0) assert_equal(cfg.getint('something', 'myint'), 3) assert_equal(cfg.getbool('something', 'myint'), True) # git demands a key without value at all to be used as a flag, thus True assert_equal(cfg.getbool('something', 'novalue'), True) assert_equal(cfg.get('something.novalue'), None) # empty value is False assert_equal(cfg.getbool('something', 'empty'), False) assert_equal(cfg.get('something.empty'), '') assert_equal(cfg.getbool('doesnot', 'exist', default=True), True) assert_raises(TypeError, cfg.getbool, 'something', 'user') # gitpython-style access assert_equal(cfg.get('something.myint'), cfg.get_value('something', 'myint')) assert_equal(cfg.get_value('doesnot', 'exist', default='oohaaa'), 'oohaaa') # weird, but that is how it is assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None) # modification follows cfg.add('something.new', 'の') assert_equal(cfg.get('something.new'), u'の') # sections are added on demand cfg.add('unheard.of', 'fame') assert_true(cfg.has_section('unheard.of')) comp = cfg.items('something') cfg.rename_section('something', 'this') assert_true(cfg.has_section('this')) assert_false(cfg.has_section('something')) # direct comparison would fail, because of section prefix assert_equal(len(cfg.items('this')), len(comp)) # fail if no such section with swallow_logs(): assert_raises(CommandError, cfg.rename_section, 'nothere', 'irrelevant') assert_true(cfg.has_option('this', 'myint')) cfg.unset('this.myint') assert_false(cfg.has_option('this', 'myint')) # batch a changes cfg.add('mike.wants.to', 'know', reload=False) assert_false('mike.wants.to' in cfg) cfg.add('mike.wants.to', 'eat') assert_true('mike.wants.to' in cfg) assert_equal(len(cfg['mike.wants.to']), 2) # set a new one: cfg.set('mike.should.have', 'known') assert_in('mike.should.have', cfg) assert_equal(cfg['mike.should.have'], 'known') # set an existing one: cfg.set('mike.should.have', 'known better') assert_equal(cfg['mike.should.have'], 'known better') # set, while there are several matching ones already: cfg.add('mike.should.have', 'a meal') assert_equal(len(cfg['mike.should.have']), 2) # raises with force=False assert_raises(CommandError, cfg.set, 'mike.should.have', 'a beer', force=False) assert_equal(len(cfg['mike.should.have']), 2) # replaces all matching ones with force=True cfg.set('mike.should.have', 'a beer', force=True) assert_equal(cfg['mike.should.have'], 'a beer') # test deprecated 'where' interface and old 'dataset' (not 'branch') value # TODO: remove along with the removal of deprecated 'where' cfg.set('mike.should.have', 'wasknown', where='dataset') assert_equal(cfg['mike.should.have'], 'wasknown') assert_equal(cfg.get_from_source('dataset', 'mike.should.have'), 'wasknown') # fails unknown location assert_raises(ValueError, cfg.add, 'somesuch', 'shit', scope='umpalumpa') # very carefully test non-local config # so carefully that even in case of bad weather Yarik doesn't find some # lame datalad unittest sections in his precious ~/.gitconfig # Note: An easier way to test this, would be to just set GIT_CONFIG_GLOBAL # to point somewhere else. However, this is not supported by git before # 2.32. Hence, stick with changed HOME in this test, but be sure to unset a # possible GIT_CONFIG_GLOBAL in addition. patched_env = os.environ.copy() patched_env.pop('GIT_CONFIG_GLOBAL', None) patched_env.update(get_home_envvars(new_home)) with patch.dict('os.environ', dict(patched_env, DATALAD_SNEAKY_ADDITION='ignore'), clear=True): global_gitconfig = opj(new_home, '.gitconfig') assert (not exists(global_gitconfig)) globalcfg = ConfigManager() assert_not_in('datalad.unittest.youcan', globalcfg) assert_in('datalad.sneaky.addition', globalcfg) cfg.add('datalad.unittest.youcan', 'removeme', scope='global') assert (exists(global_gitconfig)) # it did not go into the dataset's config! assert_not_in('datalad.unittest.youcan', cfg) # does not monitor additions! globalcfg.reload(force=True) assert_in('datalad.unittest.youcan', globalcfg) with swallow_logs(): assert_raises(CommandError, globalcfg.unset, 'datalad.unittest.youcan', scope='local') assert (globalcfg.has_section('datalad.unittest')) globalcfg.unset('datalad.unittest.youcan', scope='global') # but after we unset the only value -- that section is no longer listed assert (not globalcfg.has_section('datalad.unittest')) assert_not_in('datalad.unittest.youcan', globalcfg) ok_file_has_content(global_gitconfig, "") cfg = ConfigManager(Dataset(opj(path, 'ds')), source='branch', overrides={'datalad.godgiven': True}) assert_equal(cfg.get('datalad.godgiven'), True) # setter has no effect cfg.set('datalad.godgiven', 'false') assert_equal(cfg.get('datalad.godgiven'), True)
def _install_subds_from_flexible_source(ds, sm, **kwargs): """Tries to obtain a given subdataset from several meaningful locations Parameters ---------- ds : Dataset Parent dataset of to-be-installed subdataset. sm : dict Submodule record as produced by `subdatasets()`. **kwargs Passed onto clone() """ sm_path = op.relpath(sm['path'], start=sm['parentds']) # compose a list of candidate clone URLs clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm) # prevent inevitable exception from `clone` dest_path = op.join(ds.path, sm_path) clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path] if not clone_urls: # yield error yield get_status_dict( action='install', ds=ds, status='error', message=("Have got no candidates to install subdataset %s from.", sm_path), logger=lgr, ) return for res in clone_dataset(clone_urls_, Dataset(dest_path), **kwargs): # make sure to fix a detached HEAD before yielding the install success # result. The resetting of the branch would undo any change done # to the repo by processing in response to the result if res.get('action', None) == 'install' and \ res.get('status', None) == 'ok' and \ res.get('type', None) == 'dataset' and \ res.get('path', None) == dest_path: _fixup_submodule_dotgit_setup(ds, sm_path) # do fancy update lgr.debug( "Update cloned subdataset {0} in parent".format(dest_path)) ds.repo.update_submodule(sm_path, init=True) yield res subds = Dataset(dest_path) if not subds.is_installed(): lgr.debug('Desired subdataset %s did not materialize, stopping', subds) return # check whether clone URL generators were involved cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)] if cand_cfg: # get a handle on the configuration that is specified in the # dataset itself (local and dataset) super_cfg = ConfigManager(dataset=ds, source='dataset-local') need_reload = False for rec in cand_cfg: # check whether any of this configuration originated from the # superdataset. if so, inherit the config in the new subdataset # clone. if not, keep things clean in order to be able to move with # any outside configuration change for c in ('datalad.get.subdataset-source-candidate-{}{}'.format( rec['cost'], rec['name']), 'datalad.get.subdataset-source-candidate-{}'.format( rec['name'])): if c in super_cfg.keys(): subds.config.set(c, super_cfg.get(c), where='local', reload=False) need_reload = True break if need_reload: subds.config.reload(force=True)
def test_obtain(path): ds = create(path) cfg = ConfigManager(ds) dummy = 'datalad.test.dummy' # we know nothing and we don't know how to ask assert_raises(RuntimeError, cfg.obtain, dummy) # can report known ones cfg.add(dummy, '5.3') assert_equal(cfg.obtain(dummy), '5.3') # better type assert_equal(cfg.obtain(dummy, valtype=float), 5.3) # don't hide type issues, float doesn't become an int magically assert_raises(ValueError, cfg.obtain, dummy, valtype=int) # inject some prior knowledge from datalad.interface.common_cfg import definitions as cfg_defs cfg_defs[dummy] = dict(type=float) # no we don't need to specify a type anymore assert_equal(cfg.obtain(dummy), 5.3) # but if we remove the value from the config, all magic is gone cfg.unset(dummy) # we know nothing and we don't know how to ask assert_raises(RuntimeError, cfg.obtain, dummy) # # test actual interaction # @with_testsui() def ask(): # fail on unkown dialog type assert_raises(ValueError, cfg.obtain, dummy, dialog_type='Rorschach_test') ask() # ask nicely, and get a value of proper type using the preconfiguration @with_testsui(responses='5.3') def ask(): assert_equal( cfg.obtain(dummy, dialog_type='question', text='Tell me'), 5.3) ask() # preconfigure even more, to get the most compact call cfg_defs[dummy]['ui'] = ('question', dict(text='tell me', title='Gretchen Frage')) @with_testsui(responses='5.3') def ask(): assert_equal(cfg.obtain(dummy), 5.3) ask() @with_testsui(responses='murks') def ask(): assert_raises(ValueError, cfg.obtain, dummy) ask() # fail to store when destination is not specified, will not even ask @with_testsui() def ask(): assert_raises(ValueError, cfg.obtain, dummy, store=True) ask() # but we can preconfigure it cfg_defs[dummy]['destination'] = 'broken' @with_testsui(responses='5.3') def ask(): assert_raises(ValueError, cfg.obtain, dummy, store=True) ask() # fixup destination cfg_defs[dummy]['destination'] = 'dataset' @with_testsui(responses='5.3') def ask(): assert_equal(cfg.obtain(dummy, store=True), 5.3) ask() # now it won't have to ask again @with_testsui() def ask(): assert_equal(cfg.obtain(dummy), 5.3) ask() # wipe it out again cfg.unset(dummy) assert_not_in(dummy, cfg)
def test_something(path, new_home): # will refuse to work on dataset without a dataset assert_raises(ValueError, ConfigManager, source='dataset') # now read the example config cfg = ConfigManager(Dataset(opj(path, 'ds')), source='dataset') assert_equal(len(cfg), 5) assert_in('something.user', cfg) # multi-value assert_equal(len(cfg['something.user']), 2) assert_equal(cfg['something.user'], ('name=Jane Doe', '[email protected]')) assert_true(cfg.has_section('something')) assert_false(cfg.has_section('somethingelse')) assert_equal(sorted(cfg.sections()), [u'onemore.complicated の beast with.dot', 'something']) assert_true(cfg.has_option('something', 'user')) assert_false(cfg.has_option('something', 'us?er')) assert_false(cfg.has_option('some?thing', 'user')) assert_equal(sorted(cfg.options('something')), ['empty', 'myint', 'novalue', 'user']) assert_equal(cfg.options(u'onemore.complicated の beast with.dot'), ['findme']) assert_equal(sorted(cfg.items()), [(u'onemore.complicated の beast with.dot.findme', '5.0'), ('something.empty', ''), ('something.myint', '3'), ('something.novalue', None), ('something.user', ('name=Jane Doe', '[email protected]'))]) assert_equal(sorted(cfg.items('something')), [('something.empty', ''), ('something.myint', '3'), ('something.novalue', None), ('something.user', ('name=Jane Doe', '[email protected]'))]) # always get all values assert_equal(cfg.get('something.user'), ('name=Jane Doe', '[email protected]')) assert_raises(KeyError, cfg.__getitem__, 'somedthing.user') assert_equal( cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0) assert_equal(cfg.getint('something', 'myint'), 3) assert_equal(cfg.getbool('something', 'myint'), True) # git demands a key without value at all to be used as a flag, thus True assert_equal(cfg.getbool('something', 'novalue'), True) assert_equal(cfg.get('something.novalue'), None) # empty value is False assert_equal(cfg.getbool('something', 'empty'), False) assert_equal(cfg.get('something.empty'), '') assert_equal(cfg.getbool('doesnot', 'exist', default=True), True) assert_raises(TypeError, cfg.getbool, 'something', 'user') # gitpython-style access assert_equal(cfg.get('something.myint'), cfg.get_value('something', 'myint')) assert_equal(cfg.get_value('doesnot', 'exist', default='oohaaa'), 'oohaaa') # weired, but that is how it is assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None) # modification follows cfg.add('something.new', 'の') assert_equal(cfg.get('something.new'), u'の') # sections are added on demand cfg.add('unheard.of', 'fame') assert_true(cfg.has_section('unheard.of')) comp = cfg.items('something') cfg.rename_section('something', 'this') assert_true(cfg.has_section('this')) assert_false(cfg.has_section('something')) # direct comparision would fail, because of section prefix assert_equal(len(cfg.items('this')), len(comp)) # fail if no such section with swallow_logs(): assert_raises(CommandError, cfg.rename_section, 'nothere', 'irrelevant') assert_true(cfg.has_option('this', 'myint')) cfg.unset('this.myint') assert_false(cfg.has_option('this', 'myint')) # batch a changes cfg.add('mike.wants.to', 'know', reload=False) assert_false('mike.wants.to' in cfg) cfg.add('mike.wants.to', 'eat') assert_true('mike.wants.to' in cfg) assert_equal(len(cfg['mike.wants.to']), 2) # set a new one: cfg.set('mike.should.have', 'known') assert_in('mike.should.have', cfg) assert_equal(cfg['mike.should.have'], 'known') # set an existing one: cfg.set('mike.should.have', 'known better') assert_equal(cfg['mike.should.have'], 'known better') # set, while there are several matching ones already: cfg.add('mike.should.have', 'a meal') assert_equal(len(cfg['mike.should.have']), 2) # raises with force=False assert_raises(CommandError, cfg.set, 'mike.should.have', 'a beer', force=False) assert_equal(len(cfg['mike.should.have']), 2) # replaces all matching ones with force=True cfg.set('mike.should.have', 'a beer', force=True) assert_equal(cfg['mike.should.have'], 'a beer') # fails unknown location assert_raises(ValueError, cfg.add, 'somesuch', 'shit', where='umpalumpa') # very carefully test non-local config # so carefully that even in case of bad weather Yarik doesn't find some # lame datalad unittest sections in his precious ~/.gitconfig with patch.dict('os.environ', { 'HOME': new_home, 'DATALAD_SNEAKY_ADDITION': 'ignore' }): global_gitconfig = opj(new_home, '.gitconfig') assert (not exists(global_gitconfig)) globalcfg = ConfigManager() assert_not_in('datalad.unittest.youcan', globalcfg) assert_in('datalad.sneaky.addition', globalcfg) cfg.add('datalad.unittest.youcan', 'removeme', where='global') assert (exists(global_gitconfig)) # it did not go into the dataset's config! assert_not_in('datalad.unittest.youcan', cfg) # does not monitor additions! globalcfg.reload(force=True) assert_in('datalad.unittest.youcan', globalcfg) with swallow_logs(): assert_raises(CommandError, globalcfg.unset, 'datalad.unittest.youcan', where='local') assert (globalcfg.has_section('datalad.unittest')) globalcfg.unset('datalad.unittest.youcan', where='global') # but after we unset the only value -- that section is no longer listed assert (not globalcfg.has_section('datalad.unittest')) assert_not_in('datalad.unittest.youcan', globalcfg) if external_versions['cmd:git'] < '2.18': # older versions leave empty section behind in the file ok_file_has_content(global_gitconfig, '[datalad "unittest"]', strip=True) # remove_section to clean it up entirely globalcfg.remove_section('datalad.unittest', where='global') ok_file_has_content(global_gitconfig, "") cfg = ConfigManager(Dataset(opj(path, 'ds')), source='dataset', overrides={'datalad.godgiven': True}) assert_equal(cfg.get('datalad.godgiven'), True) # setter has no effect cfg.set('datalad.godgiven', 'false') assert_equal(cfg.get('datalad.godgiven'), True)
def test_from_env(): cfg = ConfigManager() assert_not_in('datalad.crazy.cfg', cfg) os.environ['DATALAD_CRAZY_CFG'] = 'impossibletoguess' cfg.reload() assert_in('datalad.crazy.cfg', cfg) assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess') # not in dataset-only mode cfg = ConfigManager(Dataset('nowhere'), source='dataset') assert_not_in('datalad.crazy.cfg', cfg) # check env trumps override cfg = ConfigManager() assert_not_in('datalad.crazy.override', cfg) cfg.set('datalad.crazy.override', 'fromoverride', where='override') cfg.reload() assert_equal(cfg['datalad.crazy.override'], 'fromoverride') os.environ['DATALAD_CRAZY_OVERRIDE'] = 'fromenv' cfg.reload() assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_something(path, new_home): # read nothing, has nothing cfg = ConfigManager(dataset_only=True) assert_false(len(cfg)) # now read the example config cfg = ConfigManager(Dataset(opj(path, 'ds')), dataset_only=True) assert_equal(len(cfg), 3) assert_in('something.user', cfg) # multi-value assert_equal(len(cfg['something.user']), 2) assert_equal(cfg['something.user'], ('name=Jane Doe', '[email protected]')) assert_true(cfg.has_section('something')) assert_false(cfg.has_section('somethingelse')) assert_equal(sorted(cfg.sections()), [u'onemore.complicated の beast with.dot', 'something']) assert_true(cfg.has_option('something', 'user')) assert_false(cfg.has_option('something', 'us?er')) assert_false(cfg.has_option('some?thing', 'user')) assert_equal(sorted(cfg.options('something')), ['myint', 'user']) assert_equal(cfg.options(u'onemore.complicated の beast with.dot'), ['findme']) assert_equal( sorted(cfg.items()), [(u'onemore.complicated の beast with.dot.findme', '5.0'), ('something.myint', '3'), ('something.user', ('name=Jane Doe', '[email protected]'))]) assert_equal( sorted(cfg.items('something')), [('something.myint', '3'), ('something.user', ('name=Jane Doe', '[email protected]'))]) # always get all values assert_equal( cfg.get('something.user'), ('name=Jane Doe', '[email protected]')) assert_raises(KeyError, cfg.__getitem__, 'somedthing.user') assert_equal(cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0) assert_equal(cfg.getint('something', 'myint'), 3) assert_equal(cfg.getbool('something', 'myint'), True) assert_equal(cfg.getbool('doesnot', 'exist', default=True), True) assert_raises(TypeError, cfg.getbool, 'something', 'user') # gitpython-style access assert_equal(cfg.get('something.myint'), cfg.get_value('something', 'myint')) assert_equal(cfg.get_value('doesnot', 'exist', default='oohaaa'), 'oohaaa') # weired, but that is how it is assert_raises(KeyError, cfg.get_value, 'doesnot', 'exist', default=None) # modification follows cfg.add('something.new', 'の') assert_equal(cfg.get('something.new'), u'の') # sections are added on demand cfg.add('unheard.of', 'fame') assert_true(cfg.has_section('unheard.of')) comp = cfg.items('something') cfg.rename_section('something', 'this') assert_true(cfg.has_section('this')) assert_false(cfg.has_section('something')) # direct comparision would fail, because of section prefix assert_equal(len(cfg.items('this')), len(comp)) # fail if no such section with swallow_logs(): assert_raises(CommandError, cfg.rename_section, 'nothere', 'irrelevant') assert_true(cfg.has_option('this', 'myint')) cfg.unset('this.myint') assert_false(cfg.has_option('this', 'myint')) # batch a changes cfg.add('mike.wants.to', 'know', reload=False) assert_false('mike.wants.to' in cfg) cfg.add('mike.wants.to', 'eat') assert_true('mike.wants.to' in cfg) assert_equal(len(cfg['mike.wants.to']), 2) # set a new one: cfg.set('mike.should.have', 'known') assert_in('mike.should.have', cfg) assert_equal(cfg['mike.should.have'], 'known') # set an existing one: cfg.set('mike.should.have', 'known better') assert_equal(cfg['mike.should.have'], 'known better') # set, while there are several matching ones already: cfg.add('mike.should.have', 'a meal') assert_equal(len(cfg['mike.should.have']), 2) # raises with force=False assert_raises(CommandError, cfg.set, 'mike.should.have', 'a beer', force=False) assert_equal(len(cfg['mike.should.have']), 2) # replaces all matching ones with force=True cfg.set('mike.should.have', 'a beer', force=True) assert_equal(cfg['mike.should.have'], 'a beer') # fails unknown location assert_raises(ValueError, cfg.add, 'somesuch', 'shit', where='umpalumpa') # very carefully test non-local config # so carefully that even in case of bad weather Yarik doesn't find some # lame datalad unittest sections in his precious ~/.gitconfig with patch.dict('os.environ', {'HOME': new_home, 'DATALAD_SNEAKY_ADDITION': 'ignore'}): global_gitconfig = opj(new_home, '.gitconfig') assert(not exists(global_gitconfig)) globalcfg = ConfigManager(dataset_only=False) assert_not_in('datalad.unittest.youcan', globalcfg) assert_in('datalad.sneaky.addition', globalcfg) cfg.add('datalad.unittest.youcan', 'removeme', where='global') assert(exists(global_gitconfig)) # it did not go into the dataset's config! assert_not_in('datalad.unittest.youcan', cfg) # does not monitor additions! globalcfg.reload(force=True) assert_in('datalad.unittest.youcan', globalcfg) with swallow_logs(): assert_raises( CommandError, globalcfg.unset, 'datalad.unittest.youcan', where='local') assert(globalcfg.has_section('datalad.unittest')) globalcfg.unset('datalad.unittest.youcan', where='global') # but after we unset the only value -- that section is no longer listed assert (not globalcfg.has_section('datalad.unittest')) assert_not_in('datalad.unittest.youcan', globalcfg) if external_versions['cmd:git'] < '2.18': # older versions leave empty section behind in the file ok_file_has_content(global_gitconfig, '[datalad "unittest"]', strip=True) # remove_section to clean it up entirely globalcfg.remove_section('datalad.unittest', where='global') ok_file_has_content(global_gitconfig, "") cfg = ConfigManager( Dataset(opj(path, 'ds')), dataset_only=True, overrides={'datalad.godgiven': True}) assert_equal(cfg.get('datalad.godgiven'), True) # setter has no effect cfg.set('datalad.godgiven', 'false') assert_equal(cfg.get('datalad.godgiven'), True)
def eval_func(wrapped, instance, args, kwargs): lgr.log(2, "Entered eval_func for %s", func) # for result filters # we need to produce a dict with argname/argvalue pairs for all args # incl. defaults and args given as positionals allkwargs = get_allargs_as_kwargs(wrapped, args, kwargs) # determine the command class associated with `wrapped` wrapped_class = get_wrapped_class(wrapped) # retrieve common options from kwargs, and fall back on the command # class attributes, or general defaults if needed kwargs = kwargs.copy() # we will pop, which might cause side-effect common_params = { p_name: kwargs.pop( # go with any explicitly given default p_name, # otherwise determine the command class and pull any # default set in that class getattr( wrapped_class, p_name, # or the common default eval_defaults[p_name])) for p_name in eval_params } # short cuts and configured setup for common options return_type = common_params['return_type'] result_filter = get_result_filter(common_params['result_filter']) # resolve string labels for transformers too result_xfm = known_result_xfms.get( common_params['result_xfm'], # use verbatim, if not a known label common_params['result_xfm']) result_renderer = common_params['result_renderer'] # TODO remove this conditional branch entirely, done outside if not result_renderer: result_renderer = dlcfg.get('datalad.api.result-renderer', None) # look for potential override of logging behavior result_log_level = dlcfg.get('datalad.log.result-level', None) # query cfg for defaults # .is_installed and .config can be costly, so ensure we do # it only once. See https://github.com/datalad/datalad/issues/3575 dataset_arg = allkwargs.get('dataset', None) from datalad.distribution.dataset import Dataset ds = dataset_arg if isinstance(dataset_arg, Dataset) \ else Dataset(dataset_arg) if dataset_arg else None # do not reuse a dataset's existing config manager here # they are configured to read the committed dataset configuration # too. That means a datalad update can silently bring in new # procedure definitions from the outside, and in some sense enable # remote code execution by a 3rd-party # To avoid that, create a new config manager that only reads local # config (system and .git/config), plus any overrides given to this # datalad session proc_cfg = ConfigManager( ds, source='local', overrides=dlcfg.overrides) if ds and ds.is_installed() else dlcfg # look for hooks hooks = get_jsonhooks_from_config(proc_cfg) # this internal helper function actually drives the command # generator-style, it may generate an exception if desired, # on incomplete results def generator_func(*_args, **_kwargs): # flag whether to raise an exception incomplete_results = [] # track what actions were performed how many times action_summary = {} # if a custom summary is to be provided, collect the results # of the command execution results = [] do_custom_result_summary = result_renderer in ('tailored', 'default') \ and hasattr(wrapped_class, 'custom_result_summary_renderer') # process main results for r in _process_results( # execution wrapped(*_args, **_kwargs), wrapped_class, common_params['on_failure'], # bookkeeping action_summary, incomplete_results, # communication result_renderer, result_log_level, # let renderers get to see how a command was called allkwargs): for hook, spec in hooks.items(): # run the hooks before we yield the result # this ensures that they are executed before # a potentially wrapper command gets to act # on them if match_jsonhook2result(hook, r, spec['match']): lgr.debug('Result %s matches hook %s', r, hook) # a hook is also a command that yields results # so yield them outside too # users need to pay attention to void infinite # loops, i.e. when a hook yields a result that # triggers that same hook again for hr in run_jsonhook(hook, spec, r, dataset_arg): # apply same logic as for main results, otherwise # any filters would only tackle the primary results # and a mixture of return values could happen if not keep_result(hr, result_filter, **allkwargs): continue hr = xfm_result(hr, result_xfm) # rationale for conditional is a few lines down if hr: yield hr if not keep_result(r, result_filter, **allkwargs): continue r = xfm_result(r, result_xfm) # in case the result_xfm decided to not give us anything # exclude it from the results. There is no particular reason # to do so other than that it was established behavior when # this comment was written. This will not affect any real # result record if r: yield r # collect if summary is desired if do_custom_result_summary: results.append(r) # result summary before a potential exception # custom first if do_custom_result_summary: wrapped_class.custom_result_summary_renderer(results) elif result_renderer == 'default' and action_summary and \ sum(sum(s.values()) for s in action_summary.values()) > 1: # give a summary in default mode, when there was more than one # action performed ui.message("action summary:\n {}".format('\n '.join( '{} ({})'.format( act, ', '.join( '{}: {}'.format(status, action_summary[act] [status]) for status in sorted(action_summary[act]))) for act in sorted(action_summary)))) if incomplete_results: raise IncompleteResultsError( failed=incomplete_results, msg="Command did not complete successfully") if return_type == 'generator': # hand over the generator lgr.log(2, "Returning generator_func from eval_func for %s", wrapped_class) return generator_func(*args, **kwargs) else: @wrapt.decorator def return_func(wrapped_, instance_, args_, kwargs_): results = wrapped_(*args_, **kwargs_) if inspect.isgenerator(results): # unwind generator if there is one, this actually runs # any processing results = list(results) # render summaries if not result_xfm and result_renderer in ('tailored', 'default'): # cannot render transformed results if hasattr(wrapped_class, 'custom_result_summary_renderer'): wrapped_class.custom_result_summary_renderer(results) if return_type == 'item-or-list' and \ len(results) < 2: return results[0] if results else None else: return results lgr.log(2, "Returning return_func from eval_func for %s", wrapped_class) return return_func(generator_func)(*args, **kwargs)
def _install_subds_from_flexible_source(ds, sm, **kwargs): """Tries to obtain a given subdataset from several meaningful locations Parameters ---------- ds : Dataset Parent dataset of to-be-installed subdataset. sm : dict Submodule record as produced by `subdatasets()`. **kwargs Passed onto clone() """ sm_path = op.relpath(sm['path'], start=sm['parentds']) # compose a list of candidate clone URLs clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm) # prevent inevitable exception from `clone` dest_path = op.join(ds.path, sm_path) clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path] if not clone_urls: # yield error yield get_status_dict( action='install', ds=ds, status='error', message=("Have got no candidates to install subdataset %s from.", sm_path), logger=lgr, ) return for res in clone_dataset(clone_urls_, Dataset(dest_path), cfg=ds.config, checkout_gitsha=sm['gitshasum'], **kwargs): if res.get('action', None) == 'install' and \ res.get('status', None) == 'ok' and \ res.get('type', None) == 'dataset' and \ res.get('path', None) == dest_path: _fixup_submodule_dotgit_setup(ds, sm_path) section_name = 'submodule.{}'.format(sm['gitmodule_name']) # register the submodule as "active" in the superdataset ds.config.set( '{}.active'.format(section_name), 'true', reload=False, force=True, where='local', ) ds.config.set( '{}.url'.format(section_name), # record the actual source URL of the successful clone # and not a funky prediction based on the parent ds # like ds.repo.update_submodule() would do (does not # accept a URL) res['source']['giturl'], reload=True, force=True, where='local', ) yield res subds = Dataset(dest_path) if not subds.is_installed(): lgr.debug('Desired subdataset %s did not materialize, stopping', subds) return # check whether clone URL generators were involved cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)] if cand_cfg: # get a handle on the configuration that is specified in the # dataset itself (local and dataset) super_cfg = ConfigManager(dataset=ds, source='dataset-local') need_reload = False for rec in cand_cfg: # check whether any of this configuration originated from the # superdataset. if so, inherit the config in the new subdataset # clone. if not, keep things clean in order to be able to move with # any outside configuration change for c in ('datalad.get.subdataset-source-candidate-{}{}'.format( rec['cost'], rec['name']), 'datalad.get.subdataset-source-candidate-{}'.format( rec['name'])): if c in super_cfg.keys(): subds.config.set(c, super_cfg.get(c), where='local', reload=False) need_reload = True break if need_reload: subds.config.reload(force=True)
def __init__(self, dataset, name=None, email=None, where='local'): self.config_manager = ConfigManager(dataset) self.email = email if email else SERVICE_EMAIL self.name = name if name else SERVICE_USER self.where = where
def test_crazy_cfg(path): cfg = ConfigManager(Dataset(opj(path, 'ds')), dataset_only=True) assert_in('crazy.padry', cfg)
def _install_subds_from_flexible_source(ds, sm, **kwargs): """Tries to obtain a given subdataset from several meaningful locations Parameters ---------- ds : Dataset Parent dataset of to-be-installed subdataset. sm : dict Submodule record as produced by `subdatasets()`. **kwargs Passed onto clone() """ sm_path = op.relpath(sm['path'], start=sm['parentds']) # compose a list of candidate clone URLs clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm) # prevent inevitable exception from `clone` dest_path = op.join(ds.path, sm_path) clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path] if not clone_urls: # yield error yield get_status_dict( action='install', ds=ds, status='error', message=("Have got no candidates to install subdataset %s from.", sm_path), logger=lgr, ) return for res in clone_dataset(clone_urls_, Dataset(dest_path), cfg=ds.config, **kwargs): # make sure to fix a detached HEAD before yielding the install success # result. The resetting of the branch would undo any change done # to the repo by processing in response to the result if res.get('action', None) == 'install' and \ res.get('status', None) == 'ok' and \ res.get('type', None) == 'dataset' and \ res.get('path', None) == dest_path: _fixup_submodule_dotgit_setup(ds, sm_path) target_commit = sm['gitshasum'] lgr.debug( "Update cloned subdataset {0} in parent".format(dest_path)) section_name = 'submodule.{}'.format(sm['gitmodule_name']) # do not use `git-submodule update --init`, it would make calls # to git-config which will not obey datalad inter-process locks for # modifying .git/config sub = GitRepo(res['path']) # record what branch we were on right after the clone # TODO instead of the active branch, this should first consider # a configured branch in the submodule record of the superdataset sub_orig_branch = sub.get_active_branch() # if we are on a branch this hexsha will be the tip of that branch sub_orig_hexsha = sub.get_hexsha() if sub_orig_hexsha != target_commit: # make sure we have the desired commit locally # expensive and possibly error-prone fetch conditional on cheap # local check if not sub.commit_exists(target_commit): try: sub.fetch(remote='origin', refspec=target_commit) except CommandError: pass # instead of inspecting the fetch results for possible ways # with which it could failed to produced the desired result # let's verify the presence of the commit directly, we are in # expensive-land already anyways if not sub.commit_exists(target_commit): res.update( status='error', message= ('Target commit %s does not exist in the clone, and ' 'a fetch that commit from origin failed', target_commit[:8]), ) yield res # there is nothing we can do about this # MIH thinks that removing the clone is not needed, as a likely # next step will have to be a manual recovery intervention # and not another blind attempt continue # checkout the desired commit sub.call_git(['checkout', target_commit]) # did we detach? # XXX: This is a less generic variant of a part of # GitRepo.update_submodule(). It makes use of already available # information and trusts the existence of the just cloned repo # and avoids (redoing) some safety checks if sub_orig_branch and not sub.get_active_branch(): # trace if current state is a predecessor of the branch_hexsha lgr.debug( "Detached HEAD after updating submodule %s " "(original branch: %s)", sub, sub_orig_branch) if sub.get_merge_base([sub_orig_hexsha, target_commit]) == target_commit: # TODO: config option? # MIH: There is no real need here. IMHO this should all not # happen, unless the submodule record has a branch # configured. And Datalad should leave such a record, when # a submodule is registered. # we assume the target_commit to be from the same branch, # because it is an ancestor -- update that original branch # to point to the target_commit, and update HEAD to point to # that location -- this readies the subdataset for # further modification lgr.info( "Reset subdataset branch '%s' to %s (from %s) to " "avoid a detached HEAD", sub_orig_branch, target_commit[:8], sub_orig_hexsha[:8]) branch_ref = 'refs/heads/%s' % sub_orig_branch sub.update_ref(branch_ref, target_commit) sub.update_ref('HEAD', branch_ref, symbolic=True) else: lgr.warning( "%s has a detached HEAD, because the recorded " "subdataset state %s has no unique ancestor with " "branch '%s'", sub, target_commit[:8], sub_orig_branch) # register the submodule as "active" in the superdataset ds.config.set( '{}.active'.format(section_name), 'true', reload=False, force=True, where='local', ) ds.config.set( '{}.url'.format(section_name), # record the actual source URL of the successful clone # and not a funky prediction based on the parent ds # like ds.repo.update_submodule() would do (does not # accept a URL) res['source']['giturl'], reload=True, force=True, where='local', ) yield res subds = Dataset(dest_path) if not subds.is_installed(): lgr.debug('Desired subdataset %s did not materialize, stopping', subds) return # check whether clone URL generators were involved cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)] if cand_cfg: # get a handle on the configuration that is specified in the # dataset itself (local and dataset) super_cfg = ConfigManager(dataset=ds, source='dataset-local') need_reload = False for rec in cand_cfg: # check whether any of this configuration originated from the # superdataset. if so, inherit the config in the new subdataset # clone. if not, keep things clean in order to be able to move with # any outside configuration change for c in ('datalad.get.subdataset-source-candidate-{}{}'.format( rec['cost'], rec['name']), 'datalad.get.subdataset-source-candidate-{}'.format( rec['name'])): if c in super_cfg.keys(): subds.config.set(c, super_cfg.get(c), where='local', reload=False) need_reload = True break if need_reload: subds.config.reload(force=True)