def test_url_keys(dspath, storepath):
ds = Dataset(dspath).create()
repo = ds.repo
filename = 'url_no_size.html'
# URL-type key without size
repo.call_annex([
'addurl', '--relaxed', '--raw', '--file', filename, 'http://example.com',
])
ds.save()
# copy target
ds.create_sibling_ria(
name='ria',
url='ria+file://{}'.format(storepath),
storage_sibling='only',
)
ds.get(filename)
repo.call_annex(['copy', '--to', 'ria', filename])
ds.drop(filename)
# in the store and on the web
assert_equal(len(ds.repo.whereis(filename)), 2)
# try download, but needs special permissions to even be attempted
ds.config.set('annex.security.allow-unverified-downloads', 'ACKTHPPT', where='local')
repo.call_annex(['copy', '--from', 'ria', filename])
assert_equal(len(ds.repo.whereis(filename)), 3)
# smoke tests that execute the remaining pieces with the URL key
repo.call_annex(['fsck', '-f', 'ria'])
assert_equal(len(ds.repo.whereis(filename)), 3)
# mapped key in whereis output
assert_in('%%example', repo.call_annex(['whereis', filename]))
repo.call_annex(['move', '-f', 'ria', filename])
# check that it does not magically reappear, because it actually
# did not drop the file
repo.call_annex(['fsck', '-f', 'ria'])
assert_equal(len(ds.repo.whereis(filename)), 2)
def test_no_storage(store1, store2, ds_path):
store1_url = 'ria+' + get_local_file_url(store1)
store2_url = 'ria+' + get_local_file_url(store2)
ds = Dataset(ds_path).create(force=True)
ds.save(recursive=True)
assert_repo_status(ds.path)
res = ds.create_sibling_ria(store1_url,
"datastore1",
storage_sibling=False)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_({'datastore1', 'here'},
{s['name']
for s in ds.siblings(result_renderer=None)})
# deprecated way of disabling storage still works
res = ds.create_sibling_ria(store2_url,
"datastore2",
disable_storage__=True)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_({'datastore2', 'datastore1', 'here'},
{s['name']
for s in ds.siblings(result_renderer=None)})
# smoke test that we can push to it
res = ds.push(to='datastore1')
assert_status('ok', res)
# but nothing was copied, because there is no storage sibling
assert_result_count(res, 0, action='copy')
def test_create_push_url(detection_path=None, ds_path=None, store_path=None):
store_path = Path(store_path)
ds_path = Path(ds_path)
detection_path = Path(detection_path)
ds = Dataset(ds_path).create(force=True)
ds.save()
# patch SSHConnection to signal it was used:
from datalad.support.sshconnector import SSHManager
def detector(f, d):
@wraps(f)
def _wrapper(*args, **kwargs):
d.touch()
return f(*args, **kwargs)
return _wrapper
url = "ria+{}".format(store_path.as_uri())
push_url = "ria+ssh://datalad-test{}".format(store_path.as_posix())
assert not detection_path.exists()
with patch('datalad.support.sshconnector.SSHManager.get_connection',
new=detector(SSHManager.get_connection, detection_path)):
ds.create_sibling_ria(url,
"datastore",
push_url=push_url,
new_store_ok=True)
# used ssh_manager despite file-url hence used push-url (ria+ssh):
assert detection_path.exists()
# correct config in special remote:
sr_cfg = ds.repo.get_special_remotes()[ds.siblings(
name='datastore-storage')[0]['annex-uuid']]
eq_(sr_cfg['url'], url)
eq_(sr_cfg['push-url'], push_url)
# git remote based on url (local path):
eq_(ds.config.get("remote.datastore.url"),
(store_path / ds.id[:3] / ds.id[3:]).as_posix())
eq_(
ds.config.get("remote.datastore.pushurl"),
"ssh://datalad-test{}".format(
(store_path / ds.id[:3] / ds.id[3:]).as_posix()))
# git-push uses SSH:
detection_path.unlink()
ds.push('.', to="datastore", data='nothing')
assert detection_path.exists()
# data push
# Note, that here the patching has no effect, since the special remote
# is running in a subprocess of git-annex. Hence we can't detect SSH
# usage really. However, ORA remote is tested elsewhere - if it succeeds
# all should be good wrt `create-sibling-ria`.
ds.repo.call_annex(['copy', '.', '--to', 'datastore-storage'])
def test_ephemeral(ds_path=None, store_path=None, clone_path=None):
dspath = Path(ds_path)
store = Path(store_path)
file_test = Path('file1.txt')
file_testsub = Path('sub') / 'other.txt'
# create the original dataset
ds = Dataset(dspath)
ds.create(force=True)
ds.save()
# put into store:
ds.create_sibling_ria("ria+{}".format(store.as_uri()),
"riastore",
new_store_ok=True)
ds.push(to="riastore", data="anything")
# now, get an ephemeral clone from the RIA store:
eph_clone = clone('ria+{}#{}'.format(store.as_uri(), ds.id),
clone_path,
reckless="ephemeral")
# ephemeral clone was properly linked (store has bare repos!):
clone_annex = (eph_clone.repo.dot_git / 'annex')
assert_true(clone_annex.is_symlink())
assert_true(clone_annex.resolve().samefile(store / ds.id[:3] / ds.id[3:] /
'annex'))
if not eph_clone.repo.is_managed_branch():
# TODO: We can't properly handle adjusted branch yet
# we don't need to get files in order to access them:
assert_equal((eph_clone.pathobj / file_test).read_text(), "some")
assert_equal((eph_clone.pathobj / file_testsub).read_text(), "other")
# can we unlock those files?
eph_clone.unlock(file_test)
# change content
(eph_clone.pathobj / file_test).write_text("new content")
eph_clone.save()
# new content should already be in store
# (except the store doesn't know yet)
res = eph_clone.repo.fsck(remote="riastore-storage", fast=True)
assert_equal(len(res), 2)
assert_result_count(res, 1, success=True, file=file_test.as_posix())
assert_result_count(res, 1, success=True, file=file_testsub.as_posix())
# push back git history
eph_clone.push(to=DEFAULT_REMOTE, data="nothing")
# get an update in origin
ds.update(merge=True, reobtain_data=True)
assert_equal((ds.pathobj / file_test).read_text(), "new content")
def test_no_store(path=None):
ds = Dataset(path).create()
# check that we fail without '--new-store-ok' when there is no store
assert_result_count(ds.create_sibling_ria("'ria+file:///no/where'",
"datastore",
on_failure='ignore'),
1,
status="error")
def test_create_alias(ds_path, ria_path, clone_path):
ds_path = Path(ds_path)
clone_path = Path(clone_path)
ds_path.mkdir()
dsa = Dataset(ds_path / "a").create()
res = dsa.create_sibling_ria(url="ria+file://{}".format(ria_path),
name="origin",
alias="ds-a")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
ds_clone = clone(source="ria+file://{}#~ds-a".format(ria_path),
path=clone_path / "a")
assert_repo_status(ds_clone.path)
# multiple datasets in a RIA store with different aliases work
dsb = Dataset(ds_path / "b").create()
res = dsb.create_sibling_ria(url="ria+file://{}".format(ria_path),
name="origin",
alias="ds-b")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
ds_clone = clone(source="ria+file://{}#~ds-b".format(ria_path),
path=clone_path / "b")
assert_repo_status(ds_clone.path)
# second dataset in a RIA store with the same alias emits a warning
dsc = Dataset(ds_path / "c").create()
with swallow_logs(logging.WARNING) as cml:
res = dsc.create_sibling_ria(url="ria+file://{}".format(ria_path),
name="origin",
alias="ds-a")
assert_in(
"Alias 'ds-a' already exists in the RIA store, not adding an alias",
cml.out)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
def test_no_storage(store1=None, store2=None, ds_path=None):
store1_url = 'ria+' + get_local_file_url(store1)
store2_url = 'ria+' + get_local_file_url(store2)
ds = Dataset(ds_path).create(force=True)
ds.save(recursive=True)
assert_repo_status(ds.path)
res = ds.create_sibling_ria(store1_url,
"datastore1",
storage_sibling=False,
new_store_ok=True)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_({'datastore1', 'here'},
{s['name']
for s in ds.siblings(result_renderer='disabled')})
# deprecated way of disabling storage still works
res = ds.create_sibling_ria(store2_url,
"datastore2",
storage_sibling=False,
new_store_ok=True)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_({'datastore2', 'datastore1', 'here'},
{s['name']
for s in ds.siblings(result_renderer='disabled')})
# no annex/object dir should be created when there is no special remote
# to use it.
for s in [store1, store2]:
p = Path(s) / ds.id[:3] / ds.id[3:] / 'annex' / 'objects'
assert_false(p.exists())
# smoke test that we can push to it
res = ds.push(to='datastore1')
assert_status('ok', res)
# but nothing was copied, because there is no storage sibling
assert_result_count(res, 0, action='copy')
def test_storage_only(base_path, ds_path):
store_url = 'ria+' + get_local_file_url(base_path)
ds = Dataset(ds_path).create(force=True)
ds.save(recursive=True)
assert_repo_status(ds.path)
res = ds.create_sibling_ria(store_url, "datastore", storage_sibling='only')
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
# the storage sibling uses the main name, not -storage
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'here'}, {s['name'] for s in siblings})
# smoke test that we can push to it
res = ds.push(to='datastore')
assert_status('ok', res)
assert_result_count(res, 1, action='copy')
def test_invalid_calls(path):
ds = Dataset(path).create()
# no argument:
assert_raises(TypeError, ds.create_sibling_ria)
# same name for git- and special remote:
assert_raises(ValueError,
ds.create_sibling_ria,
'ria+file:///some/where',
name='some',
storage_name='some')
# missing ria+ URL prefix
assert_result_count(
ds.create_sibling_ria('file:///some/where',
name='some',
on_failure='ignore'),
1,
status='error',
)
def _test_create_store(host, base_path, ds_path, clone_path):
ds = Dataset(ds_path).create(force=True)
subds = ds.create('sub', force=True)
subds2 = ds.create('sub2', force=True, annex=False)
ds.save(recursive=True)
assert_repo_status(ds.path)
# don't specify special remote. By default should be git-remote + "-storage"
res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
# remotes exist, but only in super
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-storage', 'here'},
{s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'here'}, {s['name'] for s in sub_siblings})
sub2_siblings = subds2.siblings(result_renderer=None)
eq_({'here'}, {s['name'] for s in sub2_siblings})
# TODO: post-update hook was enabled
# check bare repo:
git_config = Path(base_path) / ds.id[:3] / ds.id[3:] / 'config'
assert git_config.exists()
content = git_config.read_text()
assert_in("[datalad \"ora-remote\"]", content)
super_uuid = ds.config.get("remote.{}.annex-uuid".format('datastore-storage'))
assert_in("uuid = {}".format(super_uuid), content)
# implicit test of success by ria-installing from store:
ds.publish(to="datastore", transfer_data='all')
with chpwd(clone_path):
if host:
# note, we are not using the "test-store"-label here
clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id),
path='test_install')
else:
# TODO: Whenever ria+file supports special remote config (label),
# change here:
clone('ria+file://{}#{}'.format(base_path, ds.id),
path='test_install')
installed_ds = Dataset(op.join(clone_path, 'test_install'))
assert installed_ds.is_installed()
assert_repo_status(installed_ds.repo)
eq_(installed_ds.id, ds.id)
assert_in(op.join('ds', 'file1.txt'),
installed_ds.repo.get_annexed_files())
assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')),
1,
status='ok',
action='get',
path=op.join(installed_ds.path, 'ds', 'file1.txt'))
# now, again but recursive.
res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore",
recursive=True, existing='reconfigure')
eq_(len(res), 3)
assert_result_count(res, 1, path=str(ds.pathobj), status='ok', action="create-sibling-ria")
assert_result_count(res, 1, path=str(subds.pathobj), status='ok', action="create-sibling-ria")
assert_result_count(res, 1, path=str(subds2.pathobj), status='ok', action="create-sibling-ria")
# remotes now exist in super and sub
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-storage', 'here'},
{s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-storage', 'here'},
{s['name'] for s in sub_siblings})
# but no special remote in plain git subdataset:
sub2_siblings = subds2.siblings(result_renderer=None)
eq_({'datastore', 'here'},
{s['name'] for s in sub2_siblings})
# for testing trust_level parameter, redo for each label:
for trust in ['trust', 'semitrust', 'untrust']:
ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
existing='reconfigure',
trust_level=trust)
res = ds.repo.repo_info()
assert_in('[datastore-storage]',
[r['description']
for r in res['{}ed repositories'.format(trust)]])
def _test_create_store(host, base_path, ds_path, clone_path):
skip_if_no_module("ria_remote") # special remote needs to be installed
ds = Dataset(ds_path).create(force=True)
subds = ds.create('sub', force=True)
ds.save(recursive=True)
assert_repo_status(ds.path)
# don't specify special remote. By default should be git-remote + "-ria"
res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
# remotes exist, but only in super
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'here'}, {s['name'] for s in sub_siblings})
# TODO: post-update hook was enabled
# implicit test of success by ria-installing from store:
ds.publish(to="datastore", transfer_data='all')
with chpwd(clone_path):
if host:
# note, we are not using the "test-store"-label here
clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id),
path='test_install')
else:
# TODO: Whenever ria+file supports special remote config (label),
# change here:
clone('ria+file://{}#{}'.format(base_path, ds.id),
path='test_install')
installed_ds = Dataset(op.join(clone_path, 'test_install'))
assert installed_ds.is_installed()
assert_repo_status(installed_ds.repo)
eq_(installed_ds.id, ds.id)
assert_in(op.join('ds', 'file1.txt'),
installed_ds.repo.get_annexed_files())
assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')),
1,
status='ok',
action='get',
path=op.join(installed_ds.path, 'ds', 'file1.txt'))
# now, again but recursive.
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
recursive=True,
existing='reconfigure')
eq_(len(res), 2)
assert_result_count(res, 2, status='ok', action="create-sibling-ria")
# remotes now exist in super and sub
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'},
{s['name']
for s in sub_siblings})
# for testing trust_level parameter, redo for each label:
for trust in ['trust', 'semitrust', 'untrust']:
ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
existing='reconfigure',
trust_level=trust)
res = ds.repo.repo_info()
assert_in(
'[datastore-ria]',
[r['description'] for r in res['{}ed repositories'.format(trust)]])
def _test_create_store(host, ds_path, base_path, clone_path):
# TODO: This is an issue. We are writing to ~/.gitconfig here. Override
# doesn't work, since RIARemote itself (actually git-annex!) doesn't
# have access to it, so initremote will still fail.
# => at least move cfg.set/unset into a decorator, so it doesn't
# remain when a test is failing.
# TODO this should be wrapped in a decorator that performs the set/unset
# in a try-finally configuration
cfg.set('url.ria+{prot}://{host}{path}.insteadOf'
''.format(prot='ssh' if host else 'file',
host=host if host else '',
path=base_path),
'ria+ssh://test-store:',
where='global')
ds = Dataset(ds_path).create(force=True)
subds = ds.create('sub', force=True)
ds.save(recursive=True)
assert_repo_status(ds.path)
# don't specify special remote. By default should be git-remote + "-ria"
res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
# remotes exist, but only in super
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'here'}, {s['name'] for s in sub_siblings})
# TODO: post-update hook was enabled
# implicit test of success by ria-installing from store:
ds.publish(to="datastore", transfer_data='all')
with chpwd(clone_path):
if host:
# note, we are not using the "test-store"-label here
clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id),
path='test_install')
else:
# TODO: Whenever ria+file supports special remote config (label),
# change here:
clone('ria+file://{}#{}'.format(base_path, ds.id),
path='test_install')
installed_ds = Dataset(op.join(clone_path, 'test_install'))
assert installed_ds.is_installed()
assert_repo_status(installed_ds.repo)
eq_(installed_ds.id, ds.id)
assert_in(op.join('ds', 'file1.txt'),
installed_ds.repo.get_annexed_files())
assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')),
1,
status='ok',
action='get',
path=op.join(installed_ds.path, 'ds', 'file1.txt'))
# now, again but recursive.
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
recursive=True,
existing='replace')
eq_(len(res), 2)
assert_result_count(res, 2, status='ok', action="create-sibling-ria")
# remotes now exist in super and sub
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'},
{s['name']
for s in sub_siblings})
cfg.unset('url.ria+{prot}://{host}{path}.insteadOf'
''.format(prot='ssh' if host else 'file',
host=host if host else '',
path=base_path),
where='global',
reload=True)
