def test_gh2043p1(path=None): # this tests documents the interim agreement on what should happen # in the case documented in gh-2043 ds = Dataset(path).create(force=True) ds.save('1') assert_repo_status(ds.path, untracked=['2', '3']) ds.unlock('1') assert_repo_status( ds.path, # on windows we are in an unlocked branch by default, hence # we would see no change modified=[] if ds.repo.is_managed_branch() else ['1'], untracked=['2', '3']) # save(.) should recommit unlocked file, and not touch anything else # this tests the second issue in #2043 with chpwd(path): # only save modified bits save(path='.', updated=True) # state of the file (unlocked/locked) is committed as well, and the # test doesn't lock the file again assert_repo_status(ds.path, untracked=['2', '3']) with chpwd(path): # but when a path is given, anything that matches this path # untracked or not is added/saved save(path='.') # state of the file (unlocked/locked) is committed as well, and the # test doesn't lock the file again assert_repo_status(ds.path)
def test_subdataset_save(path=None): parent = Dataset(path).create() sub = parent.create('sub') assert_repo_status(parent.path) create_tree(parent.path, {"untracked": 'ignore', 'sub': {"new": "wanted"}}) sub.save('new') # defined state: one untracked, modified (but clean in itself) subdataset assert_repo_status(sub.path) assert_repo_status(parent.path, untracked=['untracked'], modified=['sub']) # `save sub` does not save the parent!! with chpwd(parent.path): assert_status('notneeded', save(dataset=sub.path)) assert_repo_status(parent.path, untracked=['untracked'], modified=['sub']) # `save -u .` saves the state change in the subdataset, # but leaves any untracked content alone with chpwd(parent.path): assert_status('ok', parent.save(updated=True)) assert_repo_status(parent.path, untracked=['untracked']) # get back to the original modified state and check that -S behaves in # exactly the same way create_tree(parent.path, {'sub': {"new2": "wanted2"}}) sub.save('new2') assert_repo_status(parent.path, untracked=['untracked'], modified=['sub'])
def test_unlock_raises(path=None, path2=None, path3=None): # make sure, we are not within a dataset: _cwd = getpwd() chpwd(path) # no dataset and no path: assert_raises(InsufficientArgumentsError, unlock, dataset=None, path=None) # no dataset and path not within a dataset: assert_raises(NoDatasetFound, unlock, dataset=None, path=path2) create(path=path, annex=False) ds = Dataset(path) # no complaints ds.unlock() # make it annex, but call unlock with invalid path: (ds.pathobj / ".noannex").unlink() AnnexRepo(path, create=True) # One that doesn't exist. res = ds.unlock(path="notexistent.txt", result_xfm=None, on_failure='ignore', return_type='item-or-list') eq_(res['message'], "path does not exist") # And one that isn't associated with a dataset. assert_in_results( ds.unlock(path=path2, on_failure="ignore"), status="error", message=("path not underneath the reference dataset %s", ds.path)) chpwd(_cwd)
def test_symlinked_relpath(path=None): # initially ran into on OSX https://github.com/datalad/datalad/issues/2406 os.makedirs(op.join(path, "origin")) dspath = op.join(path, "linked") os.symlink('origin', dspath) ds = Dataset(dspath).create() create_tree( dspath, { "mike1": 'mike1', # will be added from topdir "later": "later", # later from within subdir "d": { "mike2": 'mike2', # to be added within subdir } }) # in the root of ds with chpwd(dspath): ds.repo.add("mike1", git=True) ds.save(message="committing", path="./mike1") # Let's also do in subdirectory as CWD, check that relative path # given to a plain command (not dataset method) are treated as # relative to CWD with chpwd(op.join(dspath, 'd')): save(dataset=ds.path, message="committing", path="mike2") later = op.join(op.pardir, "later") ds.repo.add(later, git=True) save(dataset=ds.path, message="committing", path=later) assert_repo_status(dspath)
def test_subsuperdataset_save(path=None): # Verify that when invoked without recursion save does not # cause querying of subdatasets of the subdataset # see https://github.com/datalad/datalad/issues/4523 parent = Dataset(path).create() # Create 3 levels of subdatasets so later to check operation # with or without --dataset being specified sub1 = parent.create('sub1') sub2 = parent.create(sub1.pathobj / 'sub2') sub3 = parent.create(sub2.pathobj / 'sub3') assert_repo_status(path) # now we will lobotomize that sub3 so git would fail if any query is performed. (sub3.pathobj / '.git' / 'config').chmod(0o000) try: sub3.repo.call_git(['ls-files'], read_only=True) raise SkipTest except CommandError: # desired outcome pass # the call should proceed fine since neither should care about sub3 # default is no recursion parent.save('sub1') sub1.save('sub2') assert_raises(CommandError, parent.save, 'sub1', recursive=True) # and should not fail in the top level superdataset with chpwd(parent.path): save('sub1') # or in a subdataset above the problematic one with chpwd(sub1.path): save('sub2')
def test_no_leaks(path1=None, path2=None): ds1 = Dataset(path1).create() ds1.config.set('i.was.here', 'today', scope='local') assert_in('i.was.here', ds1.config.keys()) ds1.config.reload() assert_in('i.was.here', ds1.config.keys()) # now we move into this one repo, and create another # make sure that no config from ds1 leaks into ds2 with chpwd(path1): ds2 = Dataset(path2) assert_not_in('i.was.here', ds2.config.keys()) ds2.config.reload() assert_not_in('i.was.here', ds2.config.keys()) ds2.create() assert_not_in('i.was.here', ds2.config.keys()) # and that we do not track the wrong files assert_not_in(ds1.pathobj / '.git' / 'config', ds2.config._stores['git']['files']) assert_not_in(ds1.pathobj / '.datalad' / 'config', ds2.config._stores['branch']['files']) # these are the right ones assert_in(ds2.pathobj / '.git' / 'config', ds2.config._stores['git']['files']) assert_in(ds2.pathobj / '.datalad' / 'config', ds2.config._stores['branch']['files'])
def test_bf2541(path=None): ds = create(path) subds = ds.create('sub') assert_repo_status(ds.path) os.symlink('sub', op.join(ds.path, 'symlink')) with chpwd(ds.path): res = save(recursive=True) assert_repo_status(ds.path)
def test_relpath_add(path=None): ds = Dataset(path).create(force=True) with chpwd(op.join(path, 'dir')): eq_(save('testindir')[0]['path'], op.join(ds.path, 'dir', 'testindir')) # and now add all save('..') # auto-save enabled assert_repo_status(ds.path)
def test_bf2043p2(path=None): ds = Dataset(path).create(force=True) ds.repo.add('staged') assert_repo_status(ds.path, added=['staged'], untracked=['untracked']) # save -u does not commit untracked content # this tests the second issue in #2043 with chpwd(path): save(updated=True) assert_repo_status(ds.path, untracked=['untracked'])
def test_copy_file_specs_from(srcdir=None, destdir=None): srcdir = Path(srcdir) destdir = Path(destdir) files = [p for p in srcdir.glob('**/*') if not p.is_dir()] # plain list of absolute path objects r_srcabs, res = _check_copy_file_specs_from(srcdir, destdir / 'srcabs', files) # same, but with relative paths with chpwd(srcdir): r_srcrel, res = _check_copy_file_specs_from( srcdir, destdir / 'srcrel', [p.relative_to(srcdir) for p in files]) # same, but as strings r_srcabs_str, res = _check_copy_file_specs_from(srcdir, destdir / 'srcabs_str', [str(p) for p in files]) with chpwd(srcdir): r_srcrel_str, res = _check_copy_file_specs_from( srcdir, destdir / 'srcrel_str', [str(p.relative_to(srcdir)) for p in files]) # same, but with src/dest pairs r_srcdestabs_str, res = _check_copy_file_specs_from( srcdir, destdir / 'srcdestabs_str', [ '{}\0{}'.format(str(p), str(destdir / 'srcdestabs_str' / p.name)) for p in files ]) # all methods lead to the same dataset structure for a, b in ((r_srcabs, r_srcrel), (r_srcabs, r_srcabs_str), (r_srcabs, r_srcrel_str), (r_srcabs, r_srcdestabs_str)): eq_(*[ sorted(r for r in d.status(result_xfm='relpaths', result_renderer='disabled')) for d in (a, b) ]) # fail on destination outside of the dest repo res = copy_file(specs_from=[ '{}\0{}'.format(str(p), str(destdir / 'srcdest_wrong' / p.relative_to(srcdir))) for p in files ], on_failure='ignore') assert_status('error', res)
def test_not_under_git(path=None): from datalad.distribution.dataset import require_dataset dsroot = get_dataset_root(path) assert dsroot is None, "There must be no dataset above tmp %s. Got: %s" % (path, dsroot) with chpwd(path): # And require_dataset must puke also assert_raises( Exception, require_dataset, None, check_installed=True, purpose='test' )
def test_paths_with_forward_slashes(path=None): # access file with native absolute path spec print(path) ok_file_has_content(op.join(path, 'subdir', 'testfile'), 'testcontent') with chpwd(path): # native relative path spec ok_file_has_content(op.join('subdir', 'testfile'), 'testcontent') # posix relative path spec ok_file_has_content('subdir/testfile', 'testcontent') # abspath with forward slash path sep char ok_file_has_content( op.join(path, 'subdir', 'testfile').replace(op.sep, '/'), 'testcontent')
def check(ds, dataset_arg, url_file, fname_format): subdir = op.join(ds.path, "subdir") os.mkdir(subdir) with chpwd(subdir): shutil.copy(self.json_file, "in.json") addurls(url_file, "{url}", fname_format, dataset=dataset_arg, result_renderer='disabled') # Files specified in the CSV file are always relative to the # dataset. for fname in ["a", "b", "c"]: ok_exists(op.join(ds.path, fname))
def test_remove_subdataset_nomethod(path=None): ds = Dataset(path).create() ds.create('subds') with chpwd(path): # fails due to unique state res = remove('subds', on_failure='ignore') assert_in_results(res, action='uninstall', status='error', type='dataset') res = remove('subds', reckless='availability', on_failure='ignore') assert_in_results(res, action='uninstall', status='ok', type='dataset') assert_in_results(res, action='remove', status='ok') assert_in_results(res, action='save', status='ok')
def test_dataset_systemglobal_mode(path=None): ds = create(path) # any sensible (and also our CI) test environment(s) should have this assert_in('user.name', ds.config) # from .datalad/config assert_in('datalad.dataset.id', ds.config) # from .git/config assert_in('annex.version', ds.config) with chpwd(path): # now check that no config from a random dataset at PWD is picked up # if not dataset instance was provided cfg = ConfigManager(dataset=None, source='any') assert_in('user.name', cfg) assert_not_in('datalad.dataset.id', cfg) assert_not_in('annex.version', cfg)
def test_update_known_submodule(path=None): def get_baseline(p): ds = Dataset(p).create() sub = create(str(ds.pathobj / 'sub')) assert_repo_status(ds.path, untracked=['sub']) return ds # attempt one ds = get_baseline(op.join(path, 'wo_ref')) with chpwd(ds.path): save(recursive=True) assert_repo_status(ds.path) # attempt two, same as above but call add via reference dataset ds = get_baseline(op.join(path, 'w_ref')) ds.save(recursive=True) assert_repo_status(ds.path)
def test_bf1886(path=None): parent = Dataset(path).create() parent.create('sub') assert_repo_status(parent.path) # create a symlink pointing down to the subdataset, and add it os.symlink('sub', op.join(parent.path, 'down')) parent.save('down') assert_repo_status(parent.path) # now symlink pointing up os.makedirs(op.join(parent.path, 'subdir', 'subsubdir')) os.symlink(op.join(op.pardir, 'sub'), op.join(parent.path, 'subdir', 'up')) parent.save(op.join('subdir', 'up')) # 'all' to avoid the empty dir being listed assert_repo_status(parent.path, untracked_mode='all') # now symlink pointing 2xup, as in #1886 os.symlink(op.join(op.pardir, op.pardir, 'sub'), op.join(parent.path, 'subdir', 'subsubdir', 'upup')) parent.save(op.join('subdir', 'subsubdir', 'upup')) assert_repo_status(parent.path) # simulatenously add a subds and a symlink pointing to it # create subds, but don't register it create(op.join(parent.path, 'sub2')) os.symlink(op.join(op.pardir, op.pardir, 'sub2'), op.join(parent.path, 'subdir', 'subsubdir', 'upup2')) parent.save(['sub2', op.join('subdir', 'subsubdir', 'upup2')]) assert_repo_status(parent.path) # full replication of #1886: the above but be in subdir of symlink # with no reference dataset create(op.join(parent.path, 'sub3')) os.symlink(op.join(op.pardir, op.pardir, 'sub3'), op.join(parent.path, 'subdir', 'subsubdir', 'upup3')) # need to use absolute paths with chpwd(op.join(parent.path, 'subdir', 'subsubdir')): save([ op.join(parent.path, 'sub3'), op.join(parent.path, 'subdir', 'subsubdir', 'upup3') ]) assert_repo_status(parent.path)
def test_git_config_warning(path=None): if 'GIT_AUTHOR_NAME' in os.environ: raise SkipTest("Found existing explicit identity config") # Note: An easier way to test this, would be to just set GIT_CONFIG_GLOBAL # to point somewhere else. However, this is not supported by git before # 2.32. Hence, stick with changed HOME in this test, but be sure to unset a # possible GIT_CONFIG_GLOBAL in addition. patched_env = os.environ.copy() patched_env.pop('GIT_CONFIG_GLOBAL', None) patched_env.update(get_home_envvars(path)) with chpwd(path), \ patch.dict('os.environ', patched_env, clear=True), \ swallow_logs(new_level=30) as cml: # no configs in that empty HOME from datalad.api import Dataset from datalad.config import ConfigManager # reach into the class and disable the "checked" flag that # has already been tripped before we get here ConfigManager._checked_git_identity = False Dataset(path).config.reload() assert_in("configure Git before", cml.out)
def _test_create_store(host, base_path=None, ds_path=None, clone_path=None): ds = Dataset(ds_path).create(force=True) subds = ds.create('sub', force=True) subds2 = ds.create('sub2', force=True, annex=False) ds.save(recursive=True) assert_repo_status(ds.path) # don't specify special remote. By default should be git-remote + "-storage" res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore", post_update_hook=True, new_store_ok=True) assert_result_count(res, 1, status='ok', action='create-sibling-ria') # remotes exist, but only in super siblings = ds.siblings(result_renderer='disabled') eq_({'datastore', 'datastore-storage', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer='disabled') eq_({'here'}, {s['name'] for s in sub_siblings}) sub2_siblings = subds2.siblings(result_renderer='disabled') eq_({'here'}, {s['name'] for s in sub2_siblings}) # check bare repo: git_dir = Path(base_path) / ds.id[:3] / ds.id[3:] # The post-update hook was enabled. ok_exists(git_dir / "hooks" / "post-update") # And create_sibling_ria took care of an initial call to # git-update-server-info. ok_exists(git_dir / "info" / "refs") git_config = git_dir / 'config' ok_exists(git_config) content = git_config.read_text() assert_in("[datalad \"ora-remote\"]", content) super_uuid = ds.config.get( "remote.{}.annex-uuid".format('datastore-storage')) assert_in("uuid = {}".format(super_uuid), content) # implicit test of success by ria-installing from store: ds.push(to="datastore") with chpwd(clone_path): if host: # note, we are not using the "test-store"-label here clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id), path='test_install') else: # TODO: Whenever ria+file supports special remote config (label), # change here: clone('ria+file://{}#{}'.format(base_path, ds.id), path='test_install') installed_ds = Dataset(op.join(clone_path, 'test_install')) assert installed_ds.is_installed() assert_repo_status(installed_ds.repo) eq_(installed_ds.id, ds.id) # Note: get_annexed_files() always reports POSIX paths. assert_in('ds/file1.txt', installed_ds.repo.get_annexed_files()) assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')), 1, status='ok', action='get', path=op.join(installed_ds.path, 'ds', 'file1.txt')) # now, again but recursive. res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore", recursive=True, existing='reconfigure', new_store_ok=True) assert_result_count(res, 1, path=str(ds.pathobj), status='ok', action="create-sibling-ria") assert_result_count(res, 1, path=str(subds.pathobj), status='ok', action="create-sibling-ria") assert_result_count(res, 1, path=str(subds2.pathobj), status='ok', action="create-sibling-ria") # remotes now exist in super and sub siblings = ds.siblings(result_renderer='disabled') eq_({'datastore', 'datastore-storage', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer='disabled') eq_({'datastore', 'datastore-storage', 'here'}, {s['name'] for s in sub_siblings}) # but no special remote in plain git subdataset: sub2_siblings = subds2.siblings(result_renderer='disabled') eq_({'datastore', 'here'}, {s['name'] for s in sub2_siblings}) # for testing trust_level parameter, redo for each label: for trust in ['trust', 'semitrust', 'untrust']: ds.create_sibling_ria("ria+ssh://test-store:", "datastore", existing='reconfigure', trust_level=trust, new_store_ok=True) res = ds.repo.repo_info() assert_in( '[datastore-storage]', [r['description'] for r in res['{}ed repositories'.format(trust)]])
def test_save(path=None): ds = Dataset(path).create(annex=False) with open(op.join(path, "new_file.tst"), "w") as f: f.write("something") ds.repo.add("new_file.tst", git=True) ok_(ds.repo.dirty) ds.save(message="add a new file") assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) with open(op.join(path, "new_file.tst"), "w") as f: f.write("modify") ok_(ds.repo.dirty) ds.save(message="modified new_file.tst") assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) # save works without ds and files given in the PWD with open(op.join(path, "new_file.tst"), "w") as f: f.write("rapunzel") with chpwd(path): save(message="love rapunzel") assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) # and also without `-a` when things are staged with open(op.join(path, "new_file.tst"), "w") as f: f.write("exotic") ds.repo.add("new_file.tst", git=True) with chpwd(path): save(message="love marsians") assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) files = ['one.txt', 'two.txt'] for fn in files: with open(op.join(path, fn), "w") as f: f.write(fn) ds.save([op.join(path, f) for f in files]) # superfluous call to save (alll saved it already), should not fail # but report that nothing was saved assert_status('notneeded', ds.save(message="set of new files")) assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) # create subdataset subds = ds.create('subds') assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) # modify subds with open(op.join(subds.path, "some_file.tst"), "w") as f: f.write("something") subds.save() assert_repo_status(subds.path, annex=isinstance(subds.repo, AnnexRepo)) # ensure modified subds is committed ds.save() assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) # now introduce a change downstairs subds.create('someotherds') assert_repo_status(subds.path, annex=isinstance(subds.repo, AnnexRepo)) ok_(ds.repo.dirty) # and save via subdataset path ds.save('subds', version_tag='new_sub') assert_repo_status(path, annex=isinstance(ds.repo, AnnexRepo)) tags = ds.repo.get_tags() ok_(len(tags) == 1) eq_(tags[0], dict(hexsha=ds.repo.get_hexsha(), name='new_sub')) # fails when retagged, like git does res = ds.save(version_tag='new_sub', on_failure='ignore') assert_status('error', res) assert_result_count(res, 1, action='save', type='dataset', path=ds.path, message=('cannot tag this version: %s', "fatal: tag 'new_sub' already exists"))
def test_no_local_write_if_no_dataset(path=None): Dataset(path).create() with chpwd(path): cfg = ConfigManager() with assert_raises(CommandError): cfg.set('a.b.c', 'd', scope='local')
def test_path_diff(_path=None, linkpath=None): # do the setup on the real path, not the symlink, to have its # bugs not affect this test of status() ds = get_deeply_nested_structure(str(_path)) if has_symlink_capability(): # make it more complicated by default ut.Path(linkpath).symlink_to(_path, target_is_directory=True) path = linkpath else: path = _path ds = Dataset(path) if has_symlink_capability(): assert ds.pathobj != ds.repo.pathobj plain_recursive = ds.diff(recursive=True, annex='all', result_renderer='disabled') # check integrity of individual reports with a focus on how symlinks # are reported for res in plain_recursive: # anything that is an "intended" symlink should be reported # as such. In contrast, anything that is a symlink for mere # technical reasons (annex using it for something in some mode) # should be reported as the thing it is representing (i.e. # a file) if 'link2' in str(res['path']): assert res['type'] == 'symlink', res else: assert res['type'] != 'symlink', res # every item must report its parent dataset assert_in('parentds', res) # bunch of smoke tests # query of '.' is same as no path eq_( plain_recursive, ds.diff(path='.', recursive=True, annex='all', result_renderer='disabled')) # duplicate paths do not change things eq_( plain_recursive, ds.diff(path=['.', '.'], recursive=True, annex='all', result_renderer='disabled')) # neither do nested paths if not "2.24.0" <= ds.repo.git_version < "2.25.0": # Release 2.24.0 contained a regression that was fixed with 072a231016 # (2019-12-10). eq_( plain_recursive, ds.diff(path=['.', 'subds_modified'], recursive=True, annex='all', result_renderer='disabled')) # when invoked in a subdir of a dataset it still reports on the full thing # just like `git status`, as long as there are no paths specified with chpwd(op.join(path, 'directory_untracked')): plain_recursive = diff(recursive=True, annex='all', result_renderer='disabled') # should be able to take absolute paths and yield the same # output eq_( plain_recursive, ds.diff(path=ds.path, recursive=True, annex='all', result_renderer='disabled')) # query for a deeply nested path from the top, should just work with a # variety of approaches rpath = op.join('subds_modified', 'subds_lvl1_modified', u'{}_directory_untracked'.format(OBSCURE_FILENAME)) apathobj = ds.pathobj / rpath apath = str(apathobj) for p in (rpath, apath, None): if p is None: # change into the realpath of the dataset and # query with an explicit path with chpwd(ds.path): res = ds.diff(path=op.join('.', rpath), recursive=True, annex='all', result_renderer='disabled') else: res = ds.diff(path=p, recursive=True, annex='all', result_renderer='disabled') assert_result_count( res, 1, state='untracked', type='directory', refds=ds.path, # path always comes out a full path inside the queried dataset path=apath, ) assert_result_count(ds.diff(recursive=True, result_renderer='disabled'), 1, path=apath) # limiting recursion will exclude this particular path assert_result_count(ds.diff(recursive=True, recursion_limit=1, result_renderer='disabled'), 0, path=apath) # negative limit is unlimited limit eq_( ds.diff(recursive=True, recursion_limit=-1, result_renderer='disabled'), ds.diff(recursive=True, result_renderer='disabled'))
def test_wtf(topdir=None): path = opj(topdir, OBSCURE_FILENAME) # smoke test for now with swallow_outputs() as cmo: wtf(dataset=path, on_failure="ignore") assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # Those sections get sensored out by default now assert_not_in('user.name: ', cmo.out) with chpwd(path): with swallow_outputs() as cmo: wtf() assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # now with a dataset ds = create(path) with swallow_outputs() as cmo: wtf(dataset=ds.path) assert_in('## configuration', cmo.out) assert_in('## dataset', cmo.out) assert_in(u'path: {}'.format(ds.path), ensure_unicode(cmo.out)) assert_in('branches', cmo.out) assert_in(DEFAULT_BRANCH + '@', cmo.out) assert_in('git-annex@', cmo.out) # and if we run with all sensitive for sensitive in ('some', True): with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive=sensitive) # we fake those for tests anyways, but we do show cfg in this mode # and explicitly not showing them assert_in('user.name: %s' % _HIDDEN, cmo.out) with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive='all') assert_not_in(_HIDDEN, cmo.out) # all is shown assert_in('user.name: ', cmo.out) # Sections selection # # If we ask for no sections and there is no dataset with chpwd(path): with swallow_outputs() as cmo: wtf(sections=[]) assert_not_in('## dataset', cmo.out) for s in SECTION_CALLABLES: assert_not_in('## %s' % s.lower(), cmo.out.lower()) # ask for a selected set secs = ['git-annex', 'configuration'] with chpwd(path): with swallow_outputs() as cmo: wtf(sections=secs) for s in SECTION_CALLABLES: (assert_in if s in secs else assert_not_in)('## %s' % s.lower(), cmo.out.lower()) # order should match our desired one, not alphabetical # but because of https://github.com/datalad/datalad/issues/3915 # alphanum is now desired assert cmo.out.index('## git-annex') > cmo.out.index( '## configuration') # not achievable from cmdline is to pass an empty list of sections. with chpwd(path): with swallow_outputs() as cmo: wtf(sections=[]) eq_(cmo.out.rstrip(), '# WTF') # and we could decorate it nicely for embedding e.g. into github issues with swallow_outputs() as cmo: wtf(sections=['dependencies'], decor='html_details') ok_startswith(cmo.out, '<details><summary>DataLad %s WTF' % __version__) assert_in('## dependencies', cmo.out) # short flavor with swallow_outputs() as cmo: wtf(flavor='short') assert_in("- datalad: version=%s" % __version__, cmo.out) assert_in("- dependencies: ", cmo.out) eq_(len(cmo.out.splitlines()), 4) # #WTF, datalad, dependencies, trailing new line with swallow_outputs() as cmo: wtf(flavor='short', sections='*') assert_greater(len(cmo.out.splitlines()), 10) # many more # check that wtf of an unavailable section yields impossible result (#6712) res = wtf(sections=['murkie'], on_failure='ignore') eq_(res[0]["status"], "impossible") # should result only in '# WTF' skip_if_no_module('pyperclip') # verify that it works correctly in the env/platform import pyperclip with swallow_outputs() as cmo: try: pyperclip.copy("xxx") pyperclip_works = pyperclip.paste().strip() == "xxx" wtf(dataset=ds.path, clipboard=True) except (AttributeError, pyperclip.PyperclipException) as exc: # AttributeError could come from pyperclip if no DISPLAY raise SkipTest(str(exc)) assert_in("WTF information of length", cmo.out) assert_not_in('user.name', cmo.out) if not pyperclip_works: # Some times does not throw but just fails to work raise SkipTest( "Pyperclip seems to be not functioning here correctly") assert_not_in('user.name', pyperclip.paste()) assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_diff(path=None, norepo=None): with chpwd(norepo): assert_raises(NoDatasetFound, diff) ds = Dataset(path).create() assert_repo_status(ds.path) # reports stupid revision input assert_result_count(ds.diff(fr='WTF', on_failure='ignore', result_renderer='disabled'), 1, status='impossible', message="Git reference 'WTF' invalid") # no diff assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 0) assert_result_count( _dirty_results(ds.diff(fr='HEAD', result_renderer='disabled')), 0) # bogus path makes no difference assert_result_count( _dirty_results( ds.diff(path='THIS', fr='HEAD', result_renderer='disabled')), 0) # let's introduce a known change create_tree(ds.path, {'new': 'empty'}) ds.save(to_git=True) assert_repo_status(ds.path) if ds.repo.is_managed_branch(): fr_base = DEFAULT_BRANCH to = DEFAULT_BRANCH else: fr_base = "HEAD" to = None res = _dirty_results( ds.diff(fr=fr_base + '~1', to=to, result_renderer='disabled')) assert_result_count(res, 1) assert_result_count(res, 1, action='diff', path=op.join(ds.path, 'new'), state='added') # we can also find the diff without going through the dataset explicitly with chpwd(ds.path): assert_result_count(_dirty_results( diff(fr=fr_base + '~1', to=to, result_renderer='disabled')), 1, action='diff', path=op.join(ds.path, 'new'), state='added') # no diff against HEAD assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 0) # modify known file create_tree(ds.path, {'new': 'notempty'}) res = _dirty_results(ds.diff(result_renderer='disabled')) assert_result_count(res, 1) assert_result_count(res, 1, action='diff', path=op.join(ds.path, 'new'), state='modified') # but if we give another path, it doesn't show up assert_result_count(ds.diff(path='otherpath', result_renderer='disabled'), 0) # giving the right path must work though assert_result_count(ds.diff(path='new', result_renderer='disabled'), 1, action='diff', path=op.join(ds.path, 'new'), state='modified') # stage changes ds.repo.add('.', git=True) # no change in diff, staged is not committed assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 1) ds.save() assert_repo_status(ds.path) assert_result_count(_dirty_results(ds.diff(result_renderer='disabled')), 0) # untracked stuff create_tree(ds.path, {'deep': {'down': 'untracked', 'down2': 'tobeadded'}}) # a plain diff should report the untracked file # but not directly, because the parent dir is already unknown res = _dirty_results(ds.diff(result_renderer='disabled')) assert_result_count(res, 1) assert_result_count(res, 1, state='untracked', type='directory', path=op.join(ds.path, 'deep')) # report of individual files is also possible assert_result_count(ds.diff(untracked='all', result_renderer='disabled'), 2, state='untracked', type='file') # an unmatching path will hide this result assert_result_count(ds.diff(path='somewhere', result_renderer='disabled'), 0) # perfect match and anything underneath will do assert_result_count(ds.diff(path='deep', result_renderer='disabled'), 1, state='untracked', path=op.join(ds.path, 'deep'), type='directory') assert_result_count(ds.diff(path='deep', result_renderer='disabled'), 1, state='untracked', path=op.join(ds.path, 'deep')) ds.repo.add(op.join('deep', 'down2'), git=True) # now the remaining file is the only untracked one assert_result_count(ds.diff(result_renderer='disabled'), 1, state='untracked', path=op.join(ds.path, 'deep', 'down'), type='file')
def test_siblings(origin=None, repo_path=None, local_clone_path=None): ca = dict(result_renderer='disabled') # a remote dataset with a subdataset underneath origds = Dataset(origin).create(**ca) _ = origds.create('subm 1', **ca) sshurl = "ssh://push-remote.example.com" httpurl1 = "http://remote1.example.com/location" httpurl2 = "http://remote2.example.com/location" # insufficient arguments # we need a dataset to work at with chpwd(repo_path): # not yet there assert_raises(InsufficientArgumentsError, siblings, 'add', url=httpurl1, **ca) # prepare src source = install(repo_path, source=origin, recursive=True, **ca) # pollute config depvar = 'remote.test-remote.datalad-publish-depends' source.config.add(depvar, 'stupid', scope='local') # cannot configure unknown remotes as dependencies res = siblings('configure', dataset=source, name="test-remote", url=httpurl1, publish_depends=['r1', 'r2'], on_failure='ignore', **ca) assert_status('error', res) eq_(res[0]['message'], ('unknown sibling(s) specified as publication dependency: %s', set(('r1', 'r2')))) # prior config was not changed by failed call above eq_(source.config.get(depvar, None), 'stupid') res = siblings('configure', dataset=source, name="test-remote", url=httpurl1, result_xfm='paths', **ca) eq_(res, [source.path]) assert_in("test-remote", source.repo.get_remotes()) eq_(httpurl1, source.repo.get_remote_url("test-remote")) # reconfiguring doesn't change anything siblings('configure', dataset=source, name="test-remote", url=httpurl1, **ca) assert_in("test-remote", source.repo.get_remotes()) eq_(httpurl1, source.repo.get_remote_url("test-remote")) # re-adding doesn't work res = siblings('add', dataset=source, name="test-remote", url=httpurl1, on_failure='ignore', **ca) assert_status('error', res) # only after removal res = siblings('remove', dataset=source, name="test-remote", **ca) assert_status('ok', res) assert_not_in("test-remote", source.repo.get_remotes()) # remove again (with result renderer to smoke-test a renderer # special case for this too) res = siblings('remove', dataset=source, name="test-remote", **ca) assert_status('notneeded', res) res = siblings('add', dataset=source, name="test-remote", url=httpurl1, on_failure='ignore', **ca) assert_status('ok', res) # add another remove with a publication dependency # again pre-pollute config depvar = 'remote.test-remote2.datalad-publish-depends' pushvar = 'remote.test-remote2.push' source.config.add(depvar, 'stupid', scope='local') source.config.add(pushvar, 'senseless', scope='local') res = siblings( 'configure', dataset=source, name="test-remote2", url=httpurl2, on_failure='ignore', publish_depends='test-remote', # just for smoke testing publish_by_default=DEFAULT_BRANCH, **ca) assert_status('ok', res) # config replaced with new setup #source.config.reload(force=True) eq_(source.config.get(depvar, None), 'test-remote') eq_(source.config.get(pushvar, None), DEFAULT_BRANCH) # add to another remote automagically taking it from the url # and being in the dataset directory with chpwd(source.path): res = siblings('add', url=httpurl2, **ca) assert_result_count(res, 1, name="remote2.example.com", type='sibling') assert_in("remote2.example.com", source.repo.get_remotes()) # don't fail with conflicting url, when using force: res = siblings('configure', dataset=source, name="test-remote", url=httpurl1 + "/elsewhere", **ca) assert_status('ok', res) eq_(httpurl1 + "/elsewhere", source.repo.get_remote_url("test-remote")) # no longer a use case, I would need additional convincing that # this is anyhow useful other then triple checking other peoples # errors. for an actual check use 'query' # maybe it could be turned into a set of warnings when `configure` # alters an existing setting, but then why call configure, if you # want to keep the old values #with assert_raises(RuntimeError) as cm: # add_sibling(dataset=source, name="test-remote", # url=httpurl1 + "/elsewhere") #assert_in("""'test-remote' already exists with conflicting settings""", # str(cm.value)) ## add a push url without force fails, since in a way the fetch url is the ## configured push url, too, in that case: #with assert_raises(RuntimeError) as cm: # add_sibling(dataset=source, name="test-remote", # url=httpurl1 + "/elsewhere", # pushurl=sshurl, force=False) #assert_in("""'test-remote' already exists with conflicting settings""", # str(cm.value)) # add push url (force): res = siblings('configure', dataset=source, name="test-remote", url=httpurl1 + "/elsewhere", pushurl=sshurl, **ca) assert_status('ok', res) eq_(httpurl1 + "/elsewhere", source.repo.get_remote_url("test-remote")) eq_(sshurl, source.repo.get_remote_url("test-remote", push=True)) # recursively: for r in siblings( 'configure', dataset=source, name="test-remote", url=httpurl1 + "/%NAME", pushurl=sshurl + "/%NAME", recursive=True, # we need to disable annex queries, as it will try to access # the fake URL configured above get_annex_info=False, **ca): repo = GitRepo(r['path'], create=False) assert_in("test-remote", repo.get_remotes()) url = repo.get_remote_url("test-remote") pushurl = repo.get_remote_url("test-remote", push=True) ok_(url.startswith(httpurl1 + '/' + basename(source.path))) ok_(url.endswith(basename(repo.path))) ok_(pushurl.startswith(sshurl + '/' + basename(source.path))) ok_(pushurl.endswith(basename(repo.path))) eq_(url, r['url']) eq_(pushurl, r['pushurl']) # recursively without template: for r in siblings( 'configure', dataset=source, name="test-remote-2", url=httpurl1, pushurl=sshurl, recursive=True, # we need to disable annex queries, as it will try to access # the fake URL configured above get_annex_info=False, **ca): repo = GitRepo(r['path'], create=False) assert_in("test-remote-2", repo.get_remotes()) url = repo.get_remote_url("test-remote-2") pushurl = repo.get_remote_url("test-remote-2", push=True) ok_(url.startswith(httpurl1)) ok_(pushurl.startswith(sshurl)) # FIXME: next condition used to compare the *Repo objects instead of # there paths. Due to missing annex-init in # datalad/tests/utils.py:clone_url this might not be the same, since # `source` actually is an annex, but after flavor 'clone' in # `with_testrepos` and then `install` any trace of an annex might be # gone in v5 (branch 'master' only), while in direct mode it still is # considered an annex. `repo` is forced to be a `GitRepo`, so we might # compare two objects of different classes while they actually are # pointing to the same repository. # See github issue #1854 if repo.path != source.repo.path: ok_(url.endswith('/' + basename(repo.path))) ok_(pushurl.endswith(basename(repo.path))) eq_(url, r['url']) eq_(pushurl, r['pushurl']) # recursively without template and pushurl but full "hierarchy" # to a local clone for r in siblings( 'configure', dataset=source, name="test-remote-3", url=local_clone_path, recursive=True, # we need to disable annex queries, as it will try to access # the fake URL configured above get_annex_info=False, **ca): repo = GitRepo(r['path'], create=False) assert_in("test-remote-3", repo.get_remotes()) url = repo.get_remote_url("test-remote-3") pushurl = repo.get_remote_url("test-remote-3", push=True) eq_( normpath(url), normpath( opj(local_clone_path, relpath(str(r['path']), source.path)))) # https://github.com/datalad/datalad/issues/3951 ok_(not pushurl) # no pushurl should be defined # 5621: Users shouldn't pass identical names for remote & common data source assert_raises(ValueError, siblings, 'add', dataset=source, name='howdy', url=httpurl1, as_common_datasrc='howdy')