def _test_bare_git_version_2(host, dspath, store):
# Similarly to test_bare_git_version_1, this should ensure a bare git repo
# at the store location for a dataset doesn't conflict with the ORA remote.
# Note: Usability of git remote by annex depends on dataset layout version
# (dirhashlower vs. -mixed).
# For version 2 (mixed) upload via ORA and consumption via git should
# work. But not the other way around, since git-annex uses
# dirhashlower with bare repos.
ds_path = Path(dspath)
store = Path(store)
ds = Dataset(ds_path).create()
populate_dataset(ds)
ds.save()
bare_repo_path, _, _ = get_layout_locations(1, store, ds.id)
# Use git to make sure the remote end is what git thinks a bare clone of it
# should look like
subprocess.run(['git', 'clone', '--bare',
quote_cmdlinearg(str(dspath)),
quote_cmdlinearg(str(bare_repo_path))
])
if host:
url = "ria+ssh://{host}{path}".format(host=host,
path=store)
else:
url = "ria+{}".format(store.as_uri())
init_opts = common_init_opts + ['url={}'.format(url)]
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
create_store(io, store, '1')
# set up the dataset location, too.
# Note: Dataset layout version 2 (dirhash mixed):
create_ds_in_store(io, store, ds.id, '2', '1')
# Now, let's have the bare repo as a git remote
git_url = "ssh://{host}{path}".format(host=host, path=bare_repo_path) \
if host else bare_repo_path.as_uri()
ds.repo.add_remote('bare-git', git_url)
ds.repo.enable_remote('bare-git')
# and the ORA remote in addition:
ds.repo.init_remote('ora-remote', options=init_opts)
# upload keys via ORA:
ds.repo.copy_to('.', 'ora-remote')
# bare-git doesn't know yet:
eq_(len(ds.repo.whereis('one.txt')), 2)
# fsck to make availability known
assert_status(
'ok',
[annexjson2result(r, ds)
for r in ds.repo.fsck(remote='bare-git', fast=True)])
eq_(len(ds.repo.whereis('one.txt')), 3)
ds.drop('.')
eq_(len(ds.repo.whereis('one.txt')), 2)
# actually consumable via git remote:
ds.repo.call_git(['annex', 'move', 'one.txt', '--from', 'bare-git'])
eq_(len(ds.repo.whereis('one.txt')), 2)
# now, move back via git - shouldn't be consumable via ORA
ds.repo.call_git(['annex', 'move', 'one.txt', '--to', 'bare-git'])
# fsck to make availability known, but there's nothing from POV of ORA:
fsck_res = [annexjson2result(r, ds)
for r in ds.repo.fsck(remote='ora-remote', fast=True)]
assert_result_count(fsck_res,
1,
status='error',
message='** Based on the location log, one.txt\n'
'** was expected to be present, '
'but its content is missing.')
assert_result_count(fsck_res, 1, status='ok')
eq_(len(fsck_res), 2)
eq_(len(ds.repo.whereis('one.txt')), 1)
def _test_bare_git_version_1(host, dspath, store):
# This test should take a dataset and create a bare repository at the remote
# end from it.
# Given, that it is placed correctly within a tree of dataset, that remote
# thing should then be usable as an ora-remote as well as as a git-type
# remote.
# Note: Usability of git remote by annex depends on dataset layout version
# (dirhashlower vs. -mixed).
# For version 1 (lower) upload and consumption should be
# interchangeable. It doesn't matter which remote is used for what
# direction.
ds_path = Path(dspath)
store = Path(store)
ds = Dataset(ds_path).create()
populate_dataset(ds)
ds.save()
bare_repo_path, _, _ = get_layout_locations(1, store, ds.id)
# Use git to make sure the remote end is what git thinks a bare clone of it
# should look like
subprocess.run(['git', 'clone', '--bare',
quote_cmdlinearg(str(dspath)),
quote_cmdlinearg(str(bare_repo_path))
])
if host:
url = "ria+ssh://{host}{path}".format(host=host,
path=store)
else:
url = "ria+{}".format(store.as_uri())
init_opts = common_init_opts + ['url={}'.format(url)]
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
create_store(io, store, '1')
# set up the dataset location, too.
# Note: Dataset layout version 1 (dirhash lower):
create_ds_in_store(io, store, ds.id, '1', '1')
# Now, let's have the bare repo as a git remote and use it with annex
git_url = "ssh://{host}{path}".format(host=host, path=bare_repo_path) \
if host else bare_repo_path.as_uri()
ds.repo.add_remote('bare-git', git_url)
ds.repo.enable_remote('bare-git')
# copy files to the remote
ds.repo.copy_to('.', 'bare-git')
eq_(len(ds.repo.whereis('one.txt')), 2)
# now we can drop all content locally, reobtain it, and survive an
# fsck
ds.drop('.')
ds.get('.')
assert_status('ok', [annexjson2result(r, ds) for r in ds.repo.fsck()])
# Now, add the ora remote:
ds.repo.init_remote('ora-remote', options=init_opts)
# fsck to make availability known
assert_status(
'ok',
[annexjson2result(r, ds)
for r in ds.repo.fsck(remote='ora-remote', fast=True)])
eq_(len(ds.repo.whereis('one.txt')), 3)
# Now move content from git-remote to local and see it not being available
# via bare-git anymore.
ds.repo.call_git(['annex', 'move', '--all', '--from=bare-git'])
# ora-remote doesn't know yet:
eq_(len(ds.repo.whereis('one.txt')), 2)
# But after fsck it does:
fsck_res = [annexjson2result(r, ds)
for r in ds.repo.fsck(remote='ora-remote', fast=True)]
assert_result_count(fsck_res,
1,
status='error',
message='** Based on the location log, one.txt\n'
'** was expected to be present, '
'but its content is missing.')
assert_result_count(fsck_res,
1,
status='error',
message='** Based on the location log, subdir/two\n'
'** was expected to be present, '
'but its content is missing.')
eq_(len(ds.repo.whereis('one.txt')), 1)
# and the other way around: upload via ora-remote and have it available via
# git-remote:
ds.repo.copy_to('.', 'ora-remote')
# fsck to make availability known
assert_status(
'ok',
[annexjson2result(r, ds)
for r in ds.repo.fsck(remote='bare-git', fast=True)])
eq_(len(ds.repo.whereis('one.txt')), 3)
def _test_permission(host, storepath, dspath):
# Test whether ORA correctly revokes and obtains write permissions within
# the annex object tree. That is: Revoke after ORA pushed a key to store
# in order to allow the object tree to safely be used with an ephemeral
# clone. And on removal obtain write permissions, like annex would
# internally on a drop (but be sure to restore if something went wrong).
dspath = Path(dspath)
storepath = Path(storepath)
ds = Dataset(dspath).create()
populate_dataset(ds)
ds.save()
assert_repo_status(ds.path)
testfile = 'one.txt'
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host, path=storepath)
else:
store_url = "ria+{}".format(storepath.as_uri())
create_store(io, storepath, '1')
create_ds_in_store(io, storepath, ds.id, '2', '1')
_, _, obj_tree = get_layout_locations(1, storepath, ds.id)
assert_true(obj_tree.is_dir())
file_key_in_store = obj_tree / 'X9' / '6J' / 'MD5E-s8--7e55db001d319a94b0b713529a756623.txt' / 'MD5E-s8--7e55db001d319a94b0b713529a756623.txt'
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
store_uuid = ds.siblings(name='store',
return_type='item-or-list')['annex-uuid']
here_uuid = ds.siblings(name='here',
return_type='item-or-list')['annex-uuid']
known_sources = ds.repo.whereis(testfile)
assert_in(here_uuid, known_sources)
assert_not_in(store_uuid, known_sources)
assert_false(file_key_in_store.exists())
ds.repo.call_annex(['copy', testfile, '--to', 'store'])
known_sources = ds.repo.whereis(testfile)
assert_in(here_uuid, known_sources)
assert_in(store_uuid, known_sources)
assert_true(file_key_in_store.exists())
# Revoke write permissions from parent dir in-store to test whether we
# still can drop (if we can obtain the permissions). Note, that this has
# no effect on VFAT.
file_key_in_store.parent.chmod(file_key_in_store.parent.stat().st_mode
& ~stat.S_IWUSR)
# we can't directly delete; key in store should be protected
assert_raises(PermissionError, file_key_in_store.unlink)
# ORA can still drop, since it obtains permission to:
ds.repo.call_annex(['drop', testfile, '--from', 'store'])
known_sources = ds.repo.whereis(testfile)
assert_in(here_uuid, known_sources)
assert_not_in(store_uuid, known_sources)
assert_false(file_key_in_store.exists())
def _test_version_check(host, dspath, store):
dspath = Path(dspath)
store = Path(store)
ds = Dataset(dspath).create()
populate_dataset(ds)
assert_repo_status(ds.path)
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host, path=store)
else:
store_url = "ria+{}".format(store.as_uri())
create_store(io, store, '1')
# TODO: Re-establish test for version 1
# version 2: dirhash
create_ds_in_store(io, store, ds.id, '2', '1')
# add special remote
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
ds.push('.', to='store')
# check version files
remote_ds_tree_version_file = store / 'ria-layout-version'
dsgit_dir, archive_dir, dsobj_dir = \
get_layout_locations(1, store, ds.id)
remote_obj_tree_version_file = dsgit_dir / 'ria-layout-version'
assert_true(remote_ds_tree_version_file.exists())
assert_true(remote_obj_tree_version_file.exists())
with open(str(remote_ds_tree_version_file), 'r') as f:
assert_equal(f.read().strip(), '1')
with open(str(remote_obj_tree_version_file), 'r') as f:
assert_equal(f.read().strip(), '2')
# Accessing the remote should not yield any output regarding versioning,
# since it's the "correct" version. Note that "fsck" is an arbitrary choice.
# We need just something to talk to the special remote.
with swallow_logs(new_level=logging.INFO) as cml:
ds.repo.fsck(remote='store', fast=True)
# TODO: For some reason didn't get cml.assert_logged to assert
# "nothing was logged"
assert not cml.out
# Now fake-change the version
with open(str(remote_obj_tree_version_file), 'w') as f:
f.write('X\n')
# Now we should see a message about it
with swallow_logs(new_level=logging.INFO) as cml:
ds.repo.fsck(remote='store', fast=True)
cml.assert_logged(level="INFO",
msg="Remote object tree reports version X",
regex=False)
# reading still works:
ds.drop('.')
assert_status('ok', ds.get('.'))
# but writing doesn't:
with open(str(Path(ds.path) / 'new_file'), 'w') as f:
f.write("arbitrary addition")
ds.save(message="Add a new_file")
# TODO: use self.annex.error in special remote and see whether we get an
# actual error result
with assert_raises(CommandError):
ds.push('new_file', to='store')
# However, we can force it by configuration
ds.config.add("annex.ora-remote.store.force-write", "true", scope='local')
ds.push('new_file', to='store')
def _test_remote_layout(host, dspath, store, archiv_store):
dspath = Path(dspath)
store = Path(store)
archiv_store = Path(archiv_store)
ds = Dataset(dspath).create()
populate_dataset(ds)
assert_repo_status(ds.path)
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host, path=store)
arch_url = "ria+ssh://{host}{path}".format(host=host,
path=archiv_store)
else:
store_url = "ria+{}".format(store.as_uri())
arch_url = "ria+{}".format(archiv_store.as_uri())
create_store(io, store, '1')
# TODO: Re-establish test for version 1
# version 2: dirhash
create_ds_in_store(io, store, ds.id, '2', '1')
# add special remote
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
# copy files into the RIA store
ds.push('.', to='store')
# we should see the exact same annex object tree
dsgit_dir, archive_dir, dsobj_dir = \
get_layout_locations(1, store, ds.id)
store_objects = get_all_files(dsobj_dir)
local_objects = get_all_files(ds.pathobj / '.git' / 'annex' / 'objects')
assert_equal(len(store_objects), 4)
if not ds.repo.is_managed_branch():
# with managed branches the local repo uses hashdirlower instead
# TODO: However, with dataset layout version 1 this should therefore
# work on adjusted branch the same way
# TODO: Wonder whether export-archive-ora should account for that and
# rehash according to target layout.
assert_equal(sorted([p for p in store_objects]),
sorted([p for p in local_objects]))
if not io.get_7z():
raise SkipTest("No 7z available in RIA store")
# we can simply pack up the content of the remote into a
# 7z archive and place it in the right location to get a functional
# archive remote
create_store(io, archiv_store, '1')
create_ds_in_store(io, archiv_store, ds.id, '2', '1')
whereis = ds.repo.whereis('one.txt')
dsgit_dir, archive_dir, dsobj_dir = \
get_layout_locations(1, archiv_store, ds.id)
ds.export_archive_ora(archive_dir / 'archive.7z')
init_opts = common_init_opts + ['url={}'.format(arch_url)]
ds.repo.init_remote('archive', options=init_opts)
# now fsck the new remote to get the new special remote indexed
ds.repo.fsck(remote='archive', fast=True)
assert_equal(len(ds.repo.whereis('one.txt')), len(whereis) + 1)
# test creating an archive with filters on files
ds.export_archive_ora(archive_dir / 'archive2.7z',
annex_wanted='(include=*.txt)')
# test with wanted expression of a specific remote
ds.repo.set_preferred_content("wanted",
"include=subdir/*",
remote="store")
ds.export_archive_ora(archive_dir / 'archive3.7z', remote="store")
# test with the current sha
ds.export_archive_ora(
archive_dir / 'archive4.7z',
froms=ds.repo.get_revisions()[1],
)
def _postclonetest_prepare(lcl, storepath, link):
from datalad.customremotes.ria_utils import (create_store,
create_ds_in_store,
get_layout_locations)
from datalad.distributed.ora_remote import (
LocalIO, )
create_tree(lcl,
tree={
'ds': {
'test.txt': 'some',
'subdir': {
'subds': {
'testsub.txt': 'somemore'
},
'subgit': {
'testgit.txt': 'even more'
}
},
},
})
# create a local dataset with a subdataset
lcl = Path(lcl)
storepath = Path(storepath)
link = Path(link)
link.symlink_to(storepath)
subds = Dataset(lcl / 'ds' / 'subdir' / 'subds').create(force=True)
subds.save()
# add a plain git dataset as well
subgit = Dataset(lcl / 'ds' / 'subdir' / 'subgit').create(force=True,
annex=False)
subgit.save()
ds = Dataset(lcl / 'ds').create(force=True)
ds.save(version_tag='original')
assert_repo_status(ds.path)
io = LocalIO()
create_store(io, storepath, '1')
# URL to use for upload. Point is, that this should be invalid for the clone
# so that autoenable would fail. Therefore let it be based on a to be
# deleted symlink
upl_url = "ria+{}".format(get_local_file_url(str(link)))
for d in (ds, subds, subgit):
# TODO: create-sibling-ria required for config! => adapt to RF'd
# creation (missed on rebase?)
create_ds_in_store(io, storepath, d.id, '2', '1')
d.create_sibling_ria(upl_url, "store")
if d is not subgit:
# Now, simulate the problem by reconfiguring the special remote to
# not be autoenabled.
# Note, however, that the actual intention is a URL, that isn't
# valid from the point of view of the clone (doesn't resolve, no
# credentials, etc.) and therefore autoenabling on git-annex-init
# when datalad-cloning would fail to succeed.
Runner(cwd=d.path).run([
'git', 'annex', 'enableremote', 'store-storage',
'autoenable=false'
])
d.push('.', to='store')
store_loc, _, _ = get_layout_locations(1, storepath, d.id)
Runner(cwd=str(store_loc)).run(['git', 'update-server-info'])
link.unlink()
# We should now have a store with datasets that have an autoenabled ORA
# remote relying on an inaccessible URL.
# datalad-clone is supposed to reconfigure based on the URL we cloned from.
# Test this feature for cloning via HTTP, SSH and FILE URLs.
return ds.id
def get_layout_locations(base_path, dsid):
return get_layout_locations(1, base_path, dsid)