def test_url_keys(dspath, storepath): ds = Dataset(dspath).create() repo = ds.repo filename = 'url_no_size.html' # URL-type key without size repo.call_annex([ 'addurl', '--relaxed', '--raw', '--file', filename, 'http://example.com', ]) ds.save() # copy target ds.create_sibling_ria( name='ria', url='ria+file://{}'.format(storepath), storage_sibling='only', ) ds.get(filename) repo.call_annex(['copy', '--to', 'ria', filename]) ds.drop(filename) # in the store and on the web assert_equal(len(ds.repo.whereis(filename)), 2) # try download, but needs special permissions to even be attempted ds.config.set('annex.security.allow-unverified-downloads', 'ACKTHPPT', where='local') repo.call_annex(['copy', '--from', 'ria', filename]) assert_equal(len(ds.repo.whereis(filename)), 3) # smoke tests that execute the remaining pieces with the URL key repo.call_annex(['fsck', '-f', 'ria']) assert_equal(len(ds.repo.whereis(filename)), 3) # mapped key in whereis output assert_in('%%example', repo.call_annex(['whereis', filename])) repo.call_annex(['move', '-f', 'ria', filename]) # check that it does not magically reappear, because it actually # did not drop the file repo.call_annex(['fsck', '-f', 'ria']) assert_equal(len(ds.repo.whereis(filename)), 2)
def test_no_storage(store1, store2, ds_path): store1_url = 'ria+' + get_local_file_url(store1) store2_url = 'ria+' + get_local_file_url(store2) ds = Dataset(ds_path).create(force=True) ds.save(recursive=True) assert_repo_status(ds.path) res = ds.create_sibling_ria(store1_url, "datastore1", storage_sibling=False) assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_({'datastore1', 'here'}, {s['name'] for s in ds.siblings(result_renderer=None)}) # deprecated way of disabling storage still works res = ds.create_sibling_ria(store2_url, "datastore2", disable_storage__=True) assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_({'datastore2', 'datastore1', 'here'}, {s['name'] for s in ds.siblings(result_renderer=None)}) # smoke test that we can push to it res = ds.push(to='datastore1') assert_status('ok', res) # but nothing was copied, because there is no storage sibling assert_result_count(res, 0, action='copy')
def test_create_push_url(detection_path=None, ds_path=None, store_path=None): store_path = Path(store_path) ds_path = Path(ds_path) detection_path = Path(detection_path) ds = Dataset(ds_path).create(force=True) ds.save() # patch SSHConnection to signal it was used: from datalad.support.sshconnector import SSHManager def detector(f, d): @wraps(f) def _wrapper(*args, **kwargs): d.touch() return f(*args, **kwargs) return _wrapper url = "ria+{}".format(store_path.as_uri()) push_url = "ria+ssh://datalad-test{}".format(store_path.as_posix()) assert not detection_path.exists() with patch('datalad.support.sshconnector.SSHManager.get_connection', new=detector(SSHManager.get_connection, detection_path)): ds.create_sibling_ria(url, "datastore", push_url=push_url, new_store_ok=True) # used ssh_manager despite file-url hence used push-url (ria+ssh): assert detection_path.exists() # correct config in special remote: sr_cfg = ds.repo.get_special_remotes()[ds.siblings( name='datastore-storage')[0]['annex-uuid']] eq_(sr_cfg['url'], url) eq_(sr_cfg['push-url'], push_url) # git remote based on url (local path): eq_(ds.config.get("remote.datastore.url"), (store_path / ds.id[:3] / ds.id[3:]).as_posix()) eq_( ds.config.get("remote.datastore.pushurl"), "ssh://datalad-test{}".format( (store_path / ds.id[:3] / ds.id[3:]).as_posix())) # git-push uses SSH: detection_path.unlink() ds.push('.', to="datastore", data='nothing') assert detection_path.exists() # data push # Note, that here the patching has no effect, since the special remote # is running in a subprocess of git-annex. Hence we can't detect SSH # usage really. However, ORA remote is tested elsewhere - if it succeeds # all should be good wrt `create-sibling-ria`. ds.repo.call_annex(['copy', '.', '--to', 'datastore-storage'])
def test_ephemeral(ds_path=None, store_path=None, clone_path=None): dspath = Path(ds_path) store = Path(store_path) file_test = Path('file1.txt') file_testsub = Path('sub') / 'other.txt' # create the original dataset ds = Dataset(dspath) ds.create(force=True) ds.save() # put into store: ds.create_sibling_ria("ria+{}".format(store.as_uri()), "riastore", new_store_ok=True) ds.push(to="riastore", data="anything") # now, get an ephemeral clone from the RIA store: eph_clone = clone('ria+{}#{}'.format(store.as_uri(), ds.id), clone_path, reckless="ephemeral") # ephemeral clone was properly linked (store has bare repos!): clone_annex = (eph_clone.repo.dot_git / 'annex') assert_true(clone_annex.is_symlink()) assert_true(clone_annex.resolve().samefile(store / ds.id[:3] / ds.id[3:] / 'annex')) if not eph_clone.repo.is_managed_branch(): # TODO: We can't properly handle adjusted branch yet # we don't need to get files in order to access them: assert_equal((eph_clone.pathobj / file_test).read_text(), "some") assert_equal((eph_clone.pathobj / file_testsub).read_text(), "other") # can we unlock those files? eph_clone.unlock(file_test) # change content (eph_clone.pathobj / file_test).write_text("new content") eph_clone.save() # new content should already be in store # (except the store doesn't know yet) res = eph_clone.repo.fsck(remote="riastore-storage", fast=True) assert_equal(len(res), 2) assert_result_count(res, 1, success=True, file=file_test.as_posix()) assert_result_count(res, 1, success=True, file=file_testsub.as_posix()) # push back git history eph_clone.push(to=DEFAULT_REMOTE, data="nothing") # get an update in origin ds.update(merge=True, reobtain_data=True) assert_equal((ds.pathobj / file_test).read_text(), "new content")
def test_no_store(path=None): ds = Dataset(path).create() # check that we fail without '--new-store-ok' when there is no store assert_result_count(ds.create_sibling_ria("'ria+file:///no/where'", "datastore", on_failure='ignore'), 1, status="error")
def test_create_alias(ds_path, ria_path, clone_path): ds_path = Path(ds_path) clone_path = Path(clone_path) ds_path.mkdir() dsa = Dataset(ds_path / "a").create() res = dsa.create_sibling_ria(url="ria+file://{}".format(ria_path), name="origin", alias="ds-a") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) ds_clone = clone(source="ria+file://{}#~ds-a".format(ria_path), path=clone_path / "a") assert_repo_status(ds_clone.path) # multiple datasets in a RIA store with different aliases work dsb = Dataset(ds_path / "b").create() res = dsb.create_sibling_ria(url="ria+file://{}".format(ria_path), name="origin", alias="ds-b") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) ds_clone = clone(source="ria+file://{}#~ds-b".format(ria_path), path=clone_path / "b") assert_repo_status(ds_clone.path) # second dataset in a RIA store with the same alias emits a warning dsc = Dataset(ds_path / "c").create() with swallow_logs(logging.WARNING) as cml: res = dsc.create_sibling_ria(url="ria+file://{}".format(ria_path), name="origin", alias="ds-a") assert_in( "Alias 'ds-a' already exists in the RIA store, not adding an alias", cml.out) assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1)
def test_no_storage(store1=None, store2=None, ds_path=None): store1_url = 'ria+' + get_local_file_url(store1) store2_url = 'ria+' + get_local_file_url(store2) ds = Dataset(ds_path).create(force=True) ds.save(recursive=True) assert_repo_status(ds.path) res = ds.create_sibling_ria(store1_url, "datastore1", storage_sibling=False, new_store_ok=True) assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_({'datastore1', 'here'}, {s['name'] for s in ds.siblings(result_renderer='disabled')}) # deprecated way of disabling storage still works res = ds.create_sibling_ria(store2_url, "datastore2", storage_sibling=False, new_store_ok=True) assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_({'datastore2', 'datastore1', 'here'}, {s['name'] for s in ds.siblings(result_renderer='disabled')}) # no annex/object dir should be created when there is no special remote # to use it. for s in [store1, store2]: p = Path(s) / ds.id[:3] / ds.id[3:] / 'annex' / 'objects' assert_false(p.exists()) # smoke test that we can push to it res = ds.push(to='datastore1') assert_status('ok', res) # but nothing was copied, because there is no storage sibling assert_result_count(res, 0, action='copy')
def test_storage_only(base_path, ds_path): store_url = 'ria+' + get_local_file_url(base_path) ds = Dataset(ds_path).create(force=True) ds.save(recursive=True) assert_repo_status(ds.path) res = ds.create_sibling_ria(store_url, "datastore", storage_sibling='only') assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) # the storage sibling uses the main name, not -storage siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'here'}, {s['name'] for s in siblings}) # smoke test that we can push to it res = ds.push(to='datastore') assert_status('ok', res) assert_result_count(res, 1, action='copy')
def test_invalid_calls(path): ds = Dataset(path).create() # no argument: assert_raises(TypeError, ds.create_sibling_ria) # same name for git- and special remote: assert_raises(ValueError, ds.create_sibling_ria, 'ria+file:///some/where', name='some', storage_name='some') # missing ria+ URL prefix assert_result_count( ds.create_sibling_ria('file:///some/where', name='some', on_failure='ignore'), 1, status='error', )
def _test_create_store(host, base_path, ds_path, clone_path): ds = Dataset(ds_path).create(force=True) subds = ds.create('sub', force=True) subds2 = ds.create('sub2', force=True, annex=False) ds.save(recursive=True) assert_repo_status(ds.path) # don't specify special remote. By default should be git-remote + "-storage" res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) # remotes exist, but only in super siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-storage', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'here'}, {s['name'] for s in sub_siblings}) sub2_siblings = subds2.siblings(result_renderer=None) eq_({'here'}, {s['name'] for s in sub2_siblings}) # TODO: post-update hook was enabled # check bare repo: git_config = Path(base_path) / ds.id[:3] / ds.id[3:] / 'config' assert git_config.exists() content = git_config.read_text() assert_in("[datalad \"ora-remote\"]", content) super_uuid = ds.config.get("remote.{}.annex-uuid".format('datastore-storage')) assert_in("uuid = {}".format(super_uuid), content) # implicit test of success by ria-installing from store: ds.publish(to="datastore", transfer_data='all') with chpwd(clone_path): if host: # note, we are not using the "test-store"-label here clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id), path='test_install') else: # TODO: Whenever ria+file supports special remote config (label), # change here: clone('ria+file://{}#{}'.format(base_path, ds.id), path='test_install') installed_ds = Dataset(op.join(clone_path, 'test_install')) assert installed_ds.is_installed() assert_repo_status(installed_ds.repo) eq_(installed_ds.id, ds.id) assert_in(op.join('ds', 'file1.txt'), installed_ds.repo.get_annexed_files()) assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')), 1, status='ok', action='get', path=op.join(installed_ds.path, 'ds', 'file1.txt')) # now, again but recursive. res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore", recursive=True, existing='reconfigure') eq_(len(res), 3) assert_result_count(res, 1, path=str(ds.pathobj), status='ok', action="create-sibling-ria") assert_result_count(res, 1, path=str(subds.pathobj), status='ok', action="create-sibling-ria") assert_result_count(res, 1, path=str(subds2.pathobj), status='ok', action="create-sibling-ria") # remotes now exist in super and sub siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-storage', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'datastore', 'datastore-storage', 'here'}, {s['name'] for s in sub_siblings}) # but no special remote in plain git subdataset: sub2_siblings = subds2.siblings(result_renderer=None) eq_({'datastore', 'here'}, {s['name'] for s in sub2_siblings}) # for testing trust_level parameter, redo for each label: for trust in ['trust', 'semitrust', 'untrust']: ds.create_sibling_ria("ria+ssh://test-store:", "datastore", existing='reconfigure', trust_level=trust) res = ds.repo.repo_info() assert_in('[datastore-storage]', [r['description'] for r in res['{}ed repositories'.format(trust)]])
def _test_create_store(host, base_path, ds_path, clone_path): skip_if_no_module("ria_remote") # special remote needs to be installed ds = Dataset(ds_path).create(force=True) subds = ds.create('sub', force=True) ds.save(recursive=True) assert_repo_status(ds.path) # don't specify special remote. By default should be git-remote + "-ria" res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) # remotes exist, but only in super siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'here'}, {s['name'] for s in sub_siblings}) # TODO: post-update hook was enabled # implicit test of success by ria-installing from store: ds.publish(to="datastore", transfer_data='all') with chpwd(clone_path): if host: # note, we are not using the "test-store"-label here clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id), path='test_install') else: # TODO: Whenever ria+file supports special remote config (label), # change here: clone('ria+file://{}#{}'.format(base_path, ds.id), path='test_install') installed_ds = Dataset(op.join(clone_path, 'test_install')) assert installed_ds.is_installed() assert_repo_status(installed_ds.repo) eq_(installed_ds.id, ds.id) assert_in(op.join('ds', 'file1.txt'), installed_ds.repo.get_annexed_files()) assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')), 1, status='ok', action='get', path=op.join(installed_ds.path, 'ds', 'file1.txt')) # now, again but recursive. res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore", recursive=True, existing='reconfigure') eq_(len(res), 2) assert_result_count(res, 2, status='ok', action="create-sibling-ria") # remotes now exist in super and sub siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in sub_siblings}) # for testing trust_level parameter, redo for each label: for trust in ['trust', 'semitrust', 'untrust']: ds.create_sibling_ria("ria+ssh://test-store:", "datastore", existing='reconfigure', trust_level=trust) res = ds.repo.repo_info() assert_in( '[datastore-ria]', [r['description'] for r in res['{}ed repositories'.format(trust)]])
def _test_create_store(host, ds_path, base_path, clone_path): # TODO: This is an issue. We are writing to ~/.gitconfig here. Override # doesn't work, since RIARemote itself (actually git-annex!) doesn't # have access to it, so initremote will still fail. # => at least move cfg.set/unset into a decorator, so it doesn't # remain when a test is failing. # TODO this should be wrapped in a decorator that performs the set/unset # in a try-finally configuration cfg.set('url.ria+{prot}://{host}{path}.insteadOf' ''.format(prot='ssh' if host else 'file', host=host if host else '', path=base_path), 'ria+ssh://test-store:', where='global') ds = Dataset(ds_path).create(force=True) subds = ds.create('sub', force=True) ds.save(recursive=True) assert_repo_status(ds.path) # don't specify special remote. By default should be git-remote + "-ria" res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) # remotes exist, but only in super siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'here'}, {s['name'] for s in sub_siblings}) # TODO: post-update hook was enabled # implicit test of success by ria-installing from store: ds.publish(to="datastore", transfer_data='all') with chpwd(clone_path): if host: # note, we are not using the "test-store"-label here clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id), path='test_install') else: # TODO: Whenever ria+file supports special remote config (label), # change here: clone('ria+file://{}#{}'.format(base_path, ds.id), path='test_install') installed_ds = Dataset(op.join(clone_path, 'test_install')) assert installed_ds.is_installed() assert_repo_status(installed_ds.repo) eq_(installed_ds.id, ds.id) assert_in(op.join('ds', 'file1.txt'), installed_ds.repo.get_annexed_files()) assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')), 1, status='ok', action='get', path=op.join(installed_ds.path, 'ds', 'file1.txt')) # now, again but recursive. res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore", recursive=True, existing='replace') eq_(len(res), 2) assert_result_count(res, 2, status='ok', action="create-sibling-ria") # remotes now exist in super and sub siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in sub_siblings}) cfg.unset('url.ria+{prot}://{host}{path}.insteadOf' ''.format(prot='ssh' if host else 'file', host=host if host else '', path=base_path), where='global', reload=True)