def test_add_archive_use_archive_dir(repo_path):
repo = AnnexRepo(repo_path, create=True)
with chpwd(repo_path):
# Let's add first archive to the repo with default setting
archive_path = opj('4u', '1.tar.gz')
# check it gives informative error if archive is not already added
with assert_raises(RuntimeError) as cmr:
add_archive_content(archive_path)
assert_re_in(
"You should run ['\"]datalad save 4u\\\\1\\.tar\\.gz['\"] first"
if on_windows else
"You should run ['\"]datalad save 4u/1\\.tar\\.gz['\"] first",
str(cmr.exception),
match=False)
with swallow_outputs():
repo.add(archive_path)
repo.commit("added 1.tar.gz")
ok_archives_caches(repo.path, 0)
add_archive_content(archive_path,
strip_leading_dirs=True,
use_current_dir=True)
ok_(not exists(opj('4u', '1 f.txt')))
ok_file_under_git(repo.path, '1 f.txt', annexed=True)
ok_archives_caches(repo.path, 0)
# and now let's extract under archive dir
add_archive_content(archive_path, strip_leading_dirs=True)
ok_file_under_git(repo.path, opj('4u', '1 f.txt'), annexed=True)
ok_archives_caches(repo.path, 0)
add_archive_content(opj('4u', 'sub.tar.gz'))
ok_file_under_git(repo.path, opj('4u', 'sub', '2 f.txt'), annexed=True)
ok_archives_caches(repo.path, 0)
def check_compress_file(ext, annex, path, name):
# we base the archive name on the filename, in order to also
# be able to properly test compressors where the corresponding
# archive format has no capability of storing a filename
# (i.e. where the archive name itself determines the filename
# of the decompressed file, like .xz)
archive = op.join(name, _filename + ext)
compress_files([_filename], archive,
path=path)
assert_true(op.exists(archive))
if annex:
# It should work even when file is annexed and is a symlink to the
# key
from datalad.support.annexrepo import AnnexRepo
repo = AnnexRepo(path, init=True)
repo.add(_filename)
repo.commit(files=[_filename], msg="commit")
dir_extracted = name + "_extracted"
try:
decompress_file(archive, dir_extracted)
except MissingExternalDependency as exc:
raise SkipTest(exc_str(exc))
_filepath = op.join(dir_extracted, _filename)
ok_file_has_content(_filepath, 'content')
def test_proxying_open_testrepobased(repo):
TEST_CONTENT = "content to be annex-addurl'd"
fname = 'test-annex.dat'
fpath = opj(repo, fname)
assert_raises(IOError, open, fpath)
aio = AutomagicIO(activate=True)
try:
with swallow_outputs():
# now we should be able just to request to open this file
with open(fpath) as f:
content = f.read()
eq_(content, TEST_CONTENT)
finally:
aio.deactivate()
# and now that we have fetched it, nothing should forbid us to open it again
with open(fpath) as f:
eq_(f.read(), TEST_CONTENT)
annex = AnnexRepo(repo, create=False)
# Let's create another file deeper under the directory with the same content
# so it would point to the same key, which we would drop and repeat the drill
fpath2 = opj(repo, 'd1', 'd2', 'test2.dat')
os.makedirs(dirname(fpath2))
with open(fpath2, 'w') as f:
f.write(content)
annex.add(fpath2)
annex.drop(fpath2)
annex.commit("added and dropped")
assert_raises(IOError, open, fpath2)
# Let's use context manager form
with AutomagicIO() as aio:
ok_(isinstance(aio, AutomagicIO))
ok_(aio.active)
# swallowing output would cause trouble while testing with
# DATALAD_ASSERT_NO_OPEN_FILES mode on. Reason is not 100% clear
# on why underlying git-annex process would be dumping to stdout or err
#with swallow_outputs():
# now we should be able just to request to open this file
with open(fpath2) as f:
content = f.read()
eq_(content, TEST_CONTENT)
annex.drop(fpath2)
assert_raises(IOError, open, fpath2)
# Let's use relative path
with chpwd(opj(repo, 'd1')):
# Let's use context manager form
with AutomagicIO() as aio, \
swallow_outputs(), \
open(opj('d2', 'test2.dat')) as f:
content = f.read()
eq_(content, TEST_CONTENT)
def test_direct_cfg(path1, path2):
# and if repo already exists and we have env var - we fail too
# Adding backend so we get some commit into the repo
ar = AnnexRepo(path1, create=True, backend='MD5E')
del ar
AnnexRepo._unique_instances.clear() # fight flyweight
for path in (path1, path2):
with patch.dict('os.environ', {'DATALAD_REPO_DIRECT': 'True'}):
# try to create annex repo in direct mode as see how it fails
with assert_raises(DirectModeNoLongerSupportedError) as cme:
AnnexRepo(path, create=True)
assert_in("no longer supported by DataLad",
str(cme.exception)) # we have generic part
assert_in("datalad.repo.direct configuration",
str(cme.exception)) # situation specific part
# assert not op.exists(path2) # that we didn't create it - we do!
# fixing for that would be too cumbersome since we first call GitRepo.__init__
# with create
ar = AnnexRepo(path1)
# check if we somehow didn't reset the flag
assert not ar.is_direct_mode()
if ar.config.obtain("datalad.repo.version") >= 6:
raise SkipTest(
"Created repo not v5, cannot test detection of direct mode repos")
# and if repo existed before and was in direct mode, we fail too
# Since direct= option was deprecated entirely, we use protected method now
ar._set_direct_mode(True)
assert ar.is_direct_mode()
del ar # but we would need to disable somehow the flywheel
with patch.dict('os.environ', {'DATALAD_REPO_DIRECT': 'True'}):
with assert_raises(DirectModeNoLongerSupportedError) as cme:
AnnexRepo(path1, create=False)
# TODO: RM DIRECT decide what should we here -- should we test/blow?
# ATM both tests below just pass
ar2 = AnnexRepo(path2, create=True)
# happily can do it since it doesn't need a worktree to do the clone
ar2.add_submodule('sub1', url=path1)
ar2sub1 = AnnexRepo(op.join(path2, 'sub1'))
# but now let's convert that sub1 to direct mode
assert not ar2sub1.is_direct_mode()
ar2sub1._set_direct_mode(True)
assert ar2sub1.is_direct_mode()
del ar2
del ar2sub1
AnnexRepo._unique_instances.clear() # fight flyweight
ar2 = AnnexRepo(path2)
ar2.get_submodules()
# And what if we are trying to add pre-cloned repo in direct mode?
ar2sub2 = AnnexRepo.clone(path1, op.join(path2, 'sub2'))
ar2sub2._set_direct_mode(True)
del ar2sub2
AnnexRepo._unique_instances.clear() # fight flyweight
ar2.add('sub2')
def test_add_archive_content_zip(repo_path):
repo = AnnexRepo(repo_path, create=True)
with chpwd(repo_path):
with swallow_outputs():
repo.add(["1.zip"])
repo.commit("add 1.zip")
add_archive_content("1.zip")
ok_file_under_git(opj(repo.path, "1", "foo"), annexed=True)
ok_file_under_git(opj("1", "dir", "bar"), annexed=True)
ok_archives_caches(repo.path, 0)
def test_direct_cfg(path1, path2):
# and if repo already exists and we have env var - we fail too
# Adding backend so we get some commit into the repo
ar = AnnexRepo(path1, create=True, backend='MD5E')
del ar; AnnexRepo._unique_instances.clear() # fight flyweight
for path in (path1, path2):
with patch.dict('os.environ', {'DATALAD_REPO_DIRECT': 'True'}):
# try to create annex repo in direct mode as see how it fails
with assert_raises(DirectModeNoLongerSupportedError) as cme:
AnnexRepo(path, create=True)
assert_in("no longer supported by DataLad", str(cme.exception)) # we have generic part
assert_in("datalad.repo.direct configuration", str(cme.exception)) # situation specific part
# assert not op.exists(path2) # that we didn't create it - we do!
# fixing for that would be too cumbersome since we first call GitRepo.__init__
# with create
ar = AnnexRepo(path1)
# check if we somehow didn't reset the flag
assert not ar.is_direct_mode()
if ar.config.obtain("datalad.repo.version") >= 6:
raise SkipTest("Created repo not v5, cannot test detection of direct mode repos")
# and if repo existed before and was in direct mode, we fail too
# Since direct= option was deprecated entirely, we use protected method now
ar._set_direct_mode(True)
assert ar.is_direct_mode()
del ar # but we would need to disable somehow the flywheel
with patch.dict('os.environ', {'DATALAD_REPO_DIRECT': 'True'}):
with assert_raises(DirectModeNoLongerSupportedError) as cme:
AnnexRepo(path1, create=False)
# TODO: RM DIRECT decide what should we here -- should we test/blow?
# ATM both tests below just pass
ar2 = AnnexRepo(path2, create=True)
# happily can do it since it doesn't need a worktree to do the clone
ar2.add_submodule('sub1', url=path1)
ar2sub1 = AnnexRepo(op.join(path2, 'sub1'))
# but now let's convert that sub1 to direct mode
assert not ar2sub1.is_direct_mode()
ar2sub1._set_direct_mode(True)
assert ar2sub1.is_direct_mode()
del ar2; del ar2sub1; AnnexRepo._unique_instances.clear() # fight flyweight
ar2 = AnnexRepo(path2)
ar2.get_submodules()
# And what if we are trying to add pre-cloned repo in direct mode?
ar2sub2 = AnnexRepo.clone(path1, op.join(path2, 'sub2'))
ar2sub2._set_direct_mode(True)
del ar2sub2; AnnexRepo._unique_instances.clear() # fight flyweight
ar2.add('sub2')
def test_add_archive_content_absolute_path(path):
repo = AnnexRepo(opj(path, "ds"), create=True)
repo.add(["1.tar.gz"])
repo.commit("1.tar.gz")
abs_tar_gz = opj(path, "ds", "1.tar.gz")
add_archive_content(abs_tar_gz, annex=repo)
ok_file_under_git(opj(path, "ds", "1", "foo"), annexed=True)
commit_msg = repo.format_commit("%B")
# The commit message uses relative paths.
assert_not_in(abs_tar_gz, commit_msg)
assert_in("1.tar.gz", commit_msg)
with assert_raises(FileNotInRepositoryError):
add_archive_content(opj(path, "notds", "2.tar.gz"), annex=repo)
def test_get_contentlocation(tdir):
repo = AnnexRepo(tdir, create=True, init=True)
repo.add('file.dat')
repo.commit('added file.dat')
key = repo.get_file_key('file.dat')
cr = AnnexCustomRemote(tdir)
key_path = cr.get_contentlocation(key, absolute=False)
assert not isabs(key_path)
key_path_abs = cr.get_contentlocation(key, absolute=True)
assert isabs(key_path_abs)
assert cr._contentlocations == {key: key_path}
repo.drop('file.dat', options=['--force'])
assert not cr.get_contentlocation(key, absolute=True)
def test_get_contentlocation(tdir):
repo = AnnexRepo(tdir, create=True, init=True)
repo.add('file.dat')
repo.commit('added file.dat')
key = repo.get_file_key('file.dat')
cr = AnnexCustomRemote(tdir)
key_path = cr.get_contentlocation(key, absolute=False)
assert not isabs(key_path)
key_path_abs = cr.get_contentlocation(key, absolute=True)
assert isabs(key_path_abs)
assert cr._contentlocations == {key: key_path}
repo.drop('file.dat', options=['--force'])
assert not cr.get_contentlocation(key, absolute=True)
def test_get_contentlocation(tdir=None):
repo = AnnexRepo(tdir, create=True, init=True)
repo.add('file.dat')
repo.commit('added file.dat')
# TODO contentlocation would come with eval_availability=True
key = repo.get_file_annexinfo('file.dat')['key']
cr = ArchiveAnnexCustomRemote(None, path=tdir)
key_path = cr.get_contentlocation(key, absolute=False)
assert not isabs(key_path)
key_path_abs = cr.get_contentlocation(key, absolute=True)
assert isabs(key_path_abs)
assert cr._contentlocations == {key: key_path}
repo.drop('file.dat', options=['--force'])
assert not cr.get_contentlocation(key, absolute=True)
def put_file_under_git(path, filename=None, content=None, annexed=False):
"""Place file under git/annex and return used Repo
"""
annex, file_repo_path, filename, path, repo = _prep_file_under_git(path, filename)
if content is None:
content = ""
with open(opj(repo.path, file_repo_path), 'w') as f_:
f_.write(content)
if annexed:
if not isinstance(repo, AnnexRepo):
repo = AnnexRepo(repo.path)
repo.add(file_repo_path, commit=True, _datalad_msg=True)
else:
repo.add(file_repo_path, git=True, _datalad_msg=True)
ok_file_under_git(repo.path, file_repo_path, annexed)
return repo
def put_file_under_git(path, filename=None, content=None, annexed=False):
"""Place file under git/annex and return used Repo
"""
annex, file_repo_path, filename, path, repo = _prep_file_under_git(path, filename)
if content is None:
content = ""
with open(opj(repo.path, file_repo_path), 'w') as f_:
f_.write(content)
if annexed:
if not isinstance(repo, AnnexRepo):
repo = AnnexRepo(repo.path)
repo.add(file_repo_path, commit=True, _datalad_msg=True)
else:
repo.add(file_repo_path, git=True, _datalad_msg=True)
ok_file_under_git(repo.path, file_repo_path, annexed)
return repo
def test_check_dates(path):
skip_if_no_module("dateutil")
ref_ts = 1218182889 # Fri, 08 Aug 2008 04:08:09 -0400
refdate = "@{}".format(ref_ts)
repo = os.path.join(path, "repo")
with set_date(ref_ts + 5000):
ar = AnnexRepo(repo)
ar.add(".")
ar.commit()
# The standard renderer outputs json.
with swallow_outputs() as cmo:
# Set level to WARNING to avoid the progress bar when
# DATALAD_TESTS_UI_BACKEND=console.
with swallow_logs(new_level=logging.WARNING):
check_dates([repo],
reference_date=refdate,
return_type="list")
assert_in("report", json.loads(cmo.out).keys())
# We find the newer objects.
newer = call([path], reference_date=refdate)
eq_(len(newer), 1)
ok_(newer[0]["report"]["objects"])
# There are no older objects to find.
older = call([repo], reference_date=refdate, older=True)
assert_false(older[0]["report"]["objects"])
# We can pass the date in RFC 2822 format.
assert_dict_equal(
newer[0],
call([path], reference_date="08 Aug 2008 04:08:09 -0400")[0])
# paths=None defaults to the current directory.
with chpwd(path):
assert_dict_equal(
newer[0]["report"],
call(paths=None, reference_date=refdate)[0]["report"])
# Only commit type is present when annex='none'.
newer_noannex = call([path], reference_date=refdate, annex="none")
for entry in newer_noannex[0]["report"]["objects"].values():
ok_(entry["type"] == "commit")
def test_get_dataset_root(path):
eq_(get_dataset_root('/nonexistent'), None)
with chpwd(path):
repo = AnnexRepo(os.curdir, create=True)
subdir = opj('some', 'deep')
fname = opj(subdir, 'dummy')
os.makedirs(subdir)
with open(fname, 'w') as f:
f.write('some')
repo.add(fname)
# we can find this repo
eq_(get_dataset_root(os.curdir), os.curdir)
# and we get the type of path that we fed in
eq_(get_dataset_root(abspath(os.curdir)), abspath(os.curdir))
# subdirs are no issue
eq_(get_dataset_root(subdir), os.curdir)
# non-dir paths are no issue
eq_(get_dataset_root(fname), os.curdir)
def test_get_dataset_root(path):
eq_(get_dataset_root('/nonexistent'), None)
with chpwd(path):
repo = AnnexRepo(os.curdir, create=True)
subdir = opj('some', 'deep')
fname = opj(subdir, 'dummy')
os.makedirs(subdir)
with open(fname, 'w') as f:
f.write('some')
repo.add(fname)
# we can find this repo
eq_(get_dataset_root(os.curdir), os.curdir)
# and we get the type of path that we fed in
eq_(get_dataset_root(abspath(os.curdir)), abspath(os.curdir))
# subdirs are no issue
eq_(get_dataset_root(subdir), os.curdir)
# non-dir paths are no issue
eq_(get_dataset_root(fname), os.curdir)
def test_check_dates(path=None):
skip_if_no_module("dateutil")
ref_ts = 1218182889 # Fri, 08 Aug 2008 04:08:09 -0400
refdate = "@{}".format(ref_ts)
repo = os.path.join(path, "repo")
with set_date(ref_ts + 5000):
ar = AnnexRepo(repo)
ar.add(".")
ar.commit()
# The standard renderer outputs json.
with swallow_outputs() as cmo:
# Set level to WARNING to avoid the progress bar when
# DATALAD_TESTS_UI_BACKEND=console.
with swallow_logs(new_level=logging.WARNING):
check_dates([repo], reference_date=refdate, return_type="list")
assert_in("report", json.loads(cmo.out).keys())
# We find the newer objects.
newer = call([path], reference_date=refdate)
eq_(len(newer), 1)
ok_(newer[0]["report"]["objects"])
# There are no older objects to find.
older = call([repo], reference_date=refdate, older=True)
assert_false(older[0]["report"]["objects"])
# We can pass the date in RFC 2822 format.
assert_dict_equal(
newer[0],
call([path], reference_date="08 Aug 2008 04:08:09 -0400")[0])
# paths=None defaults to the current directory.
with chpwd(path):
assert_dict_equal(
newer[0]["report"],
call(paths=None, reference_date=refdate)[0]["report"])
# Only commit type is present when annex='none'.
newer_noannex = call([path], reference_date=refdate, annex="none")
for entry in newer_noannex[0]["report"]["objects"].values():
ok_(entry["type"] == "commit")
def test_update_fetch_all(src, remote_1, remote_2):
rmt1 = AnnexRepo(remote_1, src)
rmt2 = AnnexRepo(remote_2, src)
ds = Dataset(src)
ds.add_sibling(name="sibling_1", url=remote_1)
ds.add_sibling(name="sibling_2", url=remote_2)
# modify the remotes:
with open(opj(remote_1, "first.txt"), "w") as f:
f.write("some file load")
rmt1.add("first.txt", commit=True)
# TODO: Modify an already present file!
with open(opj(remote_2, "second.txt"), "w") as f:
f.write("different file load")
rmt2.add("second.txt", git=True, commit=True, msg="Add file to git.")
# fetch all remotes
ds.update(fetch_all=True)
# no merge, so changes are not in active branch:
assert_not_in("first.txt", ds.repo.get_files(ds.repo.get_active_branch()))
assert_not_in("second.txt", ds.repo.get_files(ds.repo.get_active_branch()))
# but we know the changes in remote branches:
assert_in("first.txt", ds.repo.get_files("sibling_1/master"))
assert_in("second.txt", ds.repo.get_files("sibling_2/master"))
# no merge strategy for multiple remotes yet:
assert_raises(NotImplementedError, ds.update, merge=True, fetch_all=True)
# merge a certain remote:
ds.update(name="sibling_1", merge=True)
# changes from sibling_2 still not present:
assert_not_in("second.txt", ds.repo.get_files(ds.repo.get_active_branch()))
# changes from sibling_1 merged:
assert_in("first.txt", ds.repo.get_files(ds.repo.get_active_branch()))
# it's known to annex, but has no content yet:
ds.repo.get_file_key("first.txt") # raises if unknown
eq_([False], ds.repo.file_has_content(["first.txt"]))
def test_interactions(tdir):
# Just a placeholder since constructor expects a repo
repo = AnnexRepo(tdir, create=True, init=True)
repo.add('file.dat')
repo.commit('added file.dat')
for scenario in BASE_INTERACTION_SCENARIOS + [
[
('GETAVAILABILITY', 'AVAILABILITY %s' % DEFAULT_AVAILABILITY),
('GETCOST', 'COST %d' % DEFAULT_COST),
('TRANSFER RETRIEVE somekey somefile',
re.compile('TRANSFER-FAILURE RETRIEVE somekey NotImplementedError().*')),
],
[
# by default we do not require any fancy init
# no urls supported by default
('CLAIMURL http://example.com', 'CLAIMURL-FAILURE'),
# we know that is just a single option, url, is expected so full
# one would be passed
('CLAIMURL http://example.com roguearg', 'CLAIMURL-FAILURE'),
]
]:
check_interaction_scenario(AnnexCustomRemote, tdir, scenario)
def check_compress_file(ext, annex, path, name):
archive = name + ext
compress_files([_filename], archive, path=path)
assert_true(exists(archive))
if annex:
# It should work even when file is annexed and is a symlink to the
# key
from datalad.support.annexrepo import AnnexRepo
repo = AnnexRepo(path, init=True)
repo.add(_filename)
repo.commit(files=[_filename], msg="commit")
dir_extracted = name + "_extracted"
try:
decompress_file(archive, dir_extracted)
except MissingExternalDependency as exc:
raise SkipTest(exc_str(exc))
_filepath = op.join(dir_extracted, _filename)
import glob
print(dir_extracted)
print(glob.glob(dir_extracted + '/*'))
ok_file_has_content(_filepath, 'content')
def check_compress_file(ext, annex, path, name):
archive = name + ext
compress_files([_filename], archive,
path=path)
assert_true(exists(archive))
if annex:
# It should work even when file is annexed and is a symlink to the
# key
from datalad.support.annexrepo import AnnexRepo
repo = AnnexRepo(path, init=True)
repo.add(_filename)
repo.commit(files=[_filename], msg="commit")
dir_extracted = name + "_extracted"
try:
decompress_file(archive, dir_extracted)
except MissingExternalDependency as exc:
raise SkipTest(exc_str(exc))
_filepath = op.join(dir_extracted, _filename)
import glob
print(dir_extracted)
print(glob.glob(dir_extracted + '/*'))
ok_file_has_content(_filepath, 'content')
def test_ls_json(topdir):
annex = AnnexRepo(topdir, create=True)
dsj = Dataset(topdir)
# create some file and commit it
with open(opj(dsj.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
dsj.add(path='subdsfile.txt')
dsj.save("Hello!", version_tag=1)
# add a subdataset
dsj.install('subds', source=topdir)
subdirds = dsj.create(_path_('dir/subds2'), force=True)
subdirds.add('file')
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt'),
commit=True) # commit to git to init git repo
annex.add(opj(topdir, 'dir', 'subgit'),
commit=True) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir'), commit=True) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'),
options=['--force']) # broken-link
meta_dir = opj('.git', 'datalad', 'metadata')
meta_path = opj(topdir, meta_dir)
def get_metahash(*path):
if not path:
path = ['/']
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
def get_metapath(dspath, *path):
return _path_(dspath, meta_dir, get_metahash(*path))
def get_meta(dspath, *path):
with open(get_metapath(dspath, *path)) as f:
return js.load(f)
for all_ in [True, False]: # recurse directories
for recursive in [True, False]:
for state in ['file', 'delete']:
# subdataset should have its json created and deleted when
# all=True else not
subds_metapath = get_metapath(opj(topdir, 'subds'))
exists_prior = exists(subds_metapath)
#with swallow_logs(), swallow_outputs():
dsj = _ls_json(topdir,
json=state,
all_=all_,
recursive=recursive)
exists_post = exists(subds_metapath)
# print("%s %s -> %s" % (state, exists_prior, exists_post))
assert_equal(exists_post, (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metapath = get_metapath(topdir)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metapath = get_metapath(topdir, 'dir', 'subdir')
assert_equal(exists(child_metapath),
(state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden', ), ('dir', 'subgit')]:
assert_false(exists(get_metapath(topdir, *subdir)))
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(dsj['nodes'][0]['size']['total'],
dsj['size']['total'])
# check size of subdataset
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
# dir/subds2 must not be listed among nodes of the top dataset:
topds_nodes = {x['name']: x for x in dsj['nodes']}
assert_in('subds', topds_nodes)
# XXX
# # condition here is a bit a guesswork by yoh later on
# # TODO: here and below clear destiny/interaction of all_ and recursive
# assert_equal(dsj['size']['total'],
# '15 Bytes' if (recursive and all_) else
# ('9 Bytes' if (recursive or all_) else '3 Bytes')
# )
# https://github.com/datalad/datalad/issues/1674
if state == 'file' and all_:
dirj = get_meta(topdir, 'dir')
dir_nodes = {x['name']: x for x in dirj['nodes']}
# it should be present in the subdir meta
assert_in('subds2', dir_nodes)
# and not in topds
assert_not_in('subds2', topds_nodes)
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
dsj = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
def test_ls_json(topdir):
annex = AnnexRepo(topdir, create=True)
ds = Dataset(topdir)
# create some file and commit it
with open(opj(ds.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
ds.add(path='subdsfile.txt')
ds.save("Hello!", version_tag=1)
# add a subdataset
ds.install('subds', source=topdir)
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt'),
commit=True) # commit to git to init git repo
annex.add(opj(topdir, 'dir', 'subgit'),
commit=True) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir'), commit=True) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'),
options=['--force']) # broken-link
meta_dir = opj('.git', 'datalad', 'metadata')
meta_path = opj(topdir, meta_dir)
def get_metahash(*path):
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
for all_ in [True, False]:
for recursive in [True, False]:
for state in ['file', 'delete']:
with swallow_logs(), swallow_outputs():
ds = _ls_json(topdir,
json=state,
all_=all_,
recursive=recursive)
# subdataset should have its json created and deleted when all=True else not
subds_metahash = get_metahash('/')
subds_metapath = opj(topdir, 'subds', meta_dir, subds_metahash)
assert_equal(exists(subds_metapath),
(state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metahash = get_metahash('/')
ds_metapath = opj(meta_path, ds_metahash)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metahash = get_metahash('dir', 'subdir')
child_metapath = opj(meta_path, child_metahash)
assert_equal(exists(child_metapath),
(state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden'), ('dir', 'subgit')]:
child_metahash = get_metahash(*subdir)
assert_equal(exists(opj(meta_path, child_metahash)), False)
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(ds['nodes'][0]['size']['total'],
ds['size']['total'])
# check size of subdataset
subds = [
item for item in ds['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
ds = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in ds['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
def test_check_dates(path):
refdate = 1218182889
with set_date(refdate - 1):
ar = AnnexRepo(path, create=True)
def tag_object(tag):
"""Return object for tag. Do not dereference it.
"""
# We can't use ar.get_tags because that returns the commit's hexsha,
# not the tag's, and ar.get_hexsha is limited to commit objects.
return ar.call_git_oneline(
["rev-parse", "refs/tags/{}".format(tag)], read_only=True)
ar.add("foo")
ar.commit("add foo")
foo_commit = ar.get_hexsha()
ar.commit("add foo")
ar.tag("foo-tag", "tag before refdate")
foo_tag = tag_object("foo-tag")
# Make a lightweight tag to make sure `tag_dates` doesn't choke on it.
ar.tag("light")
with set_date(refdate + 1):
ar.add("bar")
ar.commit("add bar")
bar_commit = ar.get_hexsha()
ar.tag("bar-tag", "tag after refdate")
bar_tag = tag_object("bar-tag")
with set_date(refdate + 2):
# Drop an annexed file so that we have more blobs in the git-annex
# branch than its current tree.
ar.drop("bar", options=["--force"])
results = {}
for which in ["older", "newer"]:
result = check_dates(ar, refdate, which=which)["objects"]
ok_(result)
if which == "newer":
assert_in(bar_commit, result)
assert_not_in(foo_commit, result)
assert_in(bar_tag, result)
elif which == "older":
assert_in(foo_commit, result)
assert_not_in(bar_commit, result)
assert_in(foo_tag, result)
results[which] = result
ok_(any(
x.get("filename") == "uuid.log" for x in results["older"].values()))
newer_tree = check_dates(ar, refdate, annex="tree")["objects"]
def is_annex_log_blob(entry):
return (entry["type"] == "annex-blob"
and entry["filename"].endswith(".log"))
def num_logs(entries):
return sum(map(is_annex_log_blob, entries.values()))
# Because we dropped bar above, we should have one more blob in the
# git-annex branch than in the current tree of the git-annex branch.
eq_(num_logs(results["newer"]) - num_logs(newer_tree), 1)
# Act like today is one day from the reference timestamp to check that we
# get the same results with the one-day-back default.
seconds_in_day = 60 * 60 * 24
with patch('time.time', return_value=refdate + seconds_in_day):
assert_equal(check_dates(ar, annex="tree")["objects"], newer_tree)
# We can give a path (str) instead of a GitRepo object.
assert_equal(
check_dates(path, refdate, annex="tree")["objects"], newer_tree)
with assert_raises(ValueError):
check_dates(ar, refdate, which="unrecognized")
def test_check_dates(path):
refdate = 1218182889
with set_date(refdate - 1):
ar = AnnexRepo(path, create=True)
ar.add("foo")
ar.commit("add foo")
foo_commit = ar.get_hexsha()
ar.commit("add foo")
ar.tag("foo-tag", "tag before refdate")
# We can't use ar.get_tags because that returns the commit's hexsha,
# not the tag's, and ar.get_hexsha is limited to commit objects.
foo_tag = ar.repo.git.rev_parse("foo-tag")
# Make a lightweight tag to make sure `tag_dates` doesn't choke on it.
ar.tag("light")
with set_date(refdate + 1):
ar.add("bar")
ar.commit("add bar")
bar_commit = ar.get_hexsha()
ar.tag("bar-tag", "tag after refdate")
bar_tag = ar.repo.git.rev_parse("bar-tag")
with set_date(refdate + 2):
# Drop an annexed file so that we have more blobs in the git-annex
# branch than its current tree.
ar.drop("bar", options=["--force"])
results = {}
for which in ["older", "newer"]:
result = check_dates(ar, refdate, which=which)["objects"]
ok_(result)
if which == "newer":
assert_in(bar_commit, result)
assert_not_in(foo_commit, result)
assert_in(bar_tag, result)
elif which == "older":
assert_in(foo_commit, result)
assert_not_in(bar_commit, result)
assert_in(foo_tag, result)
results[which] = result
ok_(any(x.get("filename") == "uuid.log"
for x in results["older"].values()))
newer_tree = check_dates(ar, refdate, annex="tree")["objects"]
def is_annex_log_blob(entry):
return (entry["type"] == "annex-blob"
and entry["filename"].endswith(".log"))
def num_logs(entries):
return sum(map(is_annex_log_blob, entries.values()))
# Because we dropped bar above, we should have one more blob in the
# git-annex branch than in the current tree of the git-annex branch.
eq_(num_logs(results["newer"]) - num_logs(newer_tree), 1)
# Act like today is one day from the reference timestamp to check that we
# get the same results with the one-day-back default.
seconds_in_day = 60 * 60 * 24
with patch('time.time', return_value=refdate + seconds_in_day):
assert_equal(check_dates(ar, annex="tree")["objects"],
newer_tree)
# We can give a path (str) instead of a GitRepo object.
assert_equal(check_dates(path, refdate, annex="tree")["objects"],
newer_tree)
with assert_raises(ValueError):
check_dates(ar, refdate, which="unrecognized")
def _test_proxying_open(generate_load, verify_load, repo):
annex = AnnexRepo(repo, create=True)
fpath1 = opj(repo, "test")
fpath2 = opj(repo, 'd1', 'd2', 'test2')
# generate load
fpath1 = generate_load(fpath1) or fpath1
os.makedirs(dirname(fpath2))
fpath2 = generate_load(fpath2) or fpath2
annex.add([fpath1, fpath2])
verify_load(fpath1)
verify_load(fpath2)
annex.commit("Added some files")
# clone to another repo
repo2 = repo + "_2"
annex2 = AnnexRepo.clone(repo, repo2)
# verify that can't access
fpath1_2 = fpath1.replace(repo, repo2)
fpath2_2 = fpath2.replace(repo, repo2)
EXPECTED_EXCEPTIONS = (IOError, OSError)
assert_raises(EXPECTED_EXCEPTIONS, verify_load, fpath1_2)
with AutomagicIO():
# verify that it doesn't even try to get files which do not exist
with patch('datalad.support.annexrepo.AnnexRepo.get') as gricm:
# if we request absent file
assert_raises(EXPECTED_EXCEPTIONS, open, fpath1_2 + "_", 'r')
# no get should be called
assert_false(gricm.called)
verify_load(fpath1_2)
verify_load(fpath2_2)
# and even if we drop it -- we still can get it no problem
annex2.drop(fpath2_2)
assert_false(annex2.file_has_content(fpath2_2))
verify_load(fpath2_2)
assert_true(annex2.file_has_content(fpath2_2))
annex2.drop(fpath2_2)
assert_false(annex2.file_has_content(fpath2_2))
assert_true(os.path.isfile(fpath2_2))
# In check_once mode, if we drop it, it wouldn't be considered again
annex2.drop(fpath2_2)
assert_false(annex2.file_has_content(fpath2_2))
with AutomagicIO(check_once=True):
verify_load(fpath2_2)
assert_true(annex2.file_has_content(fpath2_2))
annex2.drop(fpath2_2)
assert_false(annex2.file_has_content(fpath2_2))
assert_false(os.path.isfile(fpath2_2))
# if we override stdout with something not supporting fileno, like tornado
# does which ruins using get under IPython
# TODO: we might need to refuse any online logging in other places like that
annex2.drop(fpath2_2)
class StringIOfileno(StringIO):
def fileno(self):
raise Exception("I have no clue how to do fileno")
with patch('sys.stdout', new_callable=StringIOfileno), \
patch('sys.stderr', new_callable=StringIOfileno):
with AutomagicIO():
assert_false(annex2.file_has_content(fpath2_2))
verify_load(fpath2_2)
assert_true(annex2.file_has_content(fpath2_2))
def test_fs_traverse(topdir):
# setup temp directory tree for testing
annex = AnnexRepo(topdir)
AnnexRepo(opj(topdir, 'annexdir'), create=True)
GitRepo(opj(topdir, 'gitdir'), create=True)
GitRepo(opj(topdir, 'dir', 'subgit'), create=True)
annex.add(opj(topdir, 'dir'), commit=True)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force'])
# traverse file system in recursive and non-recursive modes
for recursive in [True, False]:
# test fs_traverse in display mode
with swallow_logs(new_level=logging.INFO) as log, swallow_outputs() as cmo:
fs = fs_traverse(topdir, AnnexRepo(topdir), recursive=recursive, json='display')
if recursive:
# fs_traverse logs should contain all not ignored subdirectories
for subdir in [opj(topdir, 'dir'), opj(topdir, 'dir', 'subdir')]:
assert_in('Directory: ' + subdir, log.out)
# fs_traverse stdout contains subdirectory
assert_in(('file2.txt' and 'dir'), cmo.out)
# extract info of the top-level child directory
child = [item for item in fs['nodes'] if item['name'] == 'dir'][0]
# size of dir type child in non-recursive modes should be 0 Bytes(default) as
# dir type child's size currently has no metadata file for traverser to pick its size from
# and would require a recursive traversal w/ write to child metadata file mode
assert_equal(child['size']['total'], {True: '6 Bytes', False: '0 Bytes'}[recursive])
for recursive in [True, False]:
# run fs_traverse in write to json 'file' mode
fs = fs_traverse(topdir, AnnexRepo(topdir), recursive=recursive, json='file')
# fs_traverse should return a dictionary
assert_equal(isinstance(fs, dict), True)
# not including git and annex folders
assert_equal([item for item in fs['nodes'] if ('gitdir' or 'annexdir') == item['name']], [])
# extract info of the top-level child directory
child = [item for item in fs['nodes'] if item['name'] == 'dir'][0]
# verify node type
assert_equal(child['type'], 'dir')
# same node size on running fs_traversal in recursive followed by non-recursive mode
# verifies child's metadata file being used to find its size
# running in reverse order (non-recursive followed by recursive mode) will give (0, actual size)
assert_equal(child['size']['total'], '6 Bytes')
# verify subdirectory traversal if run in recursive mode
if recursive:
# sub-dictionary should not include git and hidden directory info
assert_equal([item for item in child['nodes'] if ('subgit' or '.fgit') == item['name']], [])
# extract subdirectory dictionary, else fail
subchild = [subitem for subitem in child["nodes"] if subitem['name'] == 'subdir'][0]
# extract info of file1.txts, else fail
link = [subnode for subnode in subchild["nodes"] if subnode['name'] == 'file1.txt'][0]
# verify node's sizes and type
assert_equal(link['size']['total'], '3 Bytes')
assert_equal(link['size']['ondisk'], link['size']['total'])
assert_equal(link['type'], 'link')
# extract info of file2.txt, else fail
brokenlink = [subnode for subnode in subchild["nodes"] if subnode['name'] == 'file2.txt'][0]
# verify node's sizes and type
assert_equal(brokenlink['type'], 'link-broken')
assert_equal(brokenlink['size']['ondisk'], '0 Bytes')
assert_equal(brokenlink['size']['total'], '3 Bytes')
def test_ls_json(topdir, topurl):
annex = AnnexRepo(topdir, create=True)
ds = Dataset(topdir)
# create some file and commit it
with open(opj(ds.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
ds.add(path='subdsfile.txt')
ds.save("Hello!", version_tag=1)
# add a subdataset
ds.install('subds', source=topdir)
subdirds = ds.create(_path_('dir/subds2'), force=True)
subdirds.add('file')
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt')) # commit to git to init git repo
git.commit()
annex.add(opj(topdir, 'dir', 'subgit')) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir')) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force']) # broken-link
annex.commit()
git.add('fgit.txt') # commit to git to init git repo
git.commit()
# annex.add doesn't add submodule, so using ds.add
ds.add(opj('dir', 'subgit')) # add the non-dataset git repo to annex
ds.add('dir') # add to annex (links)
ds.drop(opj('dir', 'subdir', 'file2.txt'), check=False) # broken-link
# register "external" submodule by installing and uninstalling it
ext_url = topurl + '/dir/subgit/.git'
# need to make it installable via http
Runner()('git update-server-info', cwd=opj(topdir, 'dir', 'subgit'))
ds.install(opj('dir', 'subgit_ext'), source=ext_url)
ds.uninstall(opj('dir', 'subgit_ext'))
meta_dir = opj('.git', 'datalad', 'metadata')
def get_metahash(*path):
if not path:
path = ['/']
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
def get_metapath(dspath, *path):
return _path_(dspath, meta_dir, get_metahash(*path))
def get_meta(dspath, *path):
with open(get_metapath(dspath, *path)) as f:
return js.load(f)
# Let's see that there is no crash if one of the files is available only
# in relaxed URL mode, so no size could be picked up
ds.repo.add_url_to_file(
'fromweb', topurl + '/noteventhere', options=['--relaxed'])
for all_ in [True, False]: # recurse directories
for recursive in [True, False]:
for state in ['file', 'delete']:
# subdataset should have its json created and deleted when
# all=True else not
subds_metapath = get_metapath(opj(topdir, 'subds'))
exists_prior = exists(subds_metapath)
#with swallow_logs(), swallow_outputs():
dsj = _ls_json(
topdir,
json=state,
all_=all_,
recursive=recursive
)
ok_startswith(dsj['tags'], '1-')
exists_post = exists(subds_metapath)
# print("%s %s -> %s" % (state, exists_prior, exists_post))
assert_equal(exists_post, (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metapath = get_metapath(topdir)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metapath = get_metapath(topdir, 'dir', 'subdir')
assert_equal(exists(child_metapath), (state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden',), ('dir', 'subgit')]:
assert_false(exists(get_metapath(topdir, *subdir)))
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(dsj['nodes'][0]['size']['total'], dsj['size']['total'])
# check size of subdataset
subds = [item for item in dsj['nodes'] if item['name'] == ('subdsfile.txt' or 'subds')][0]
assert_equal(subds['size']['total'], '3 Bytes')
# dir/subds2 must not be listed among nodes of the top dataset:
topds_nodes = {x['name']: x for x in dsj['nodes']}
assert_in('subds', topds_nodes)
# XXX
# # condition here is a bit a guesswork by yoh later on
# # TODO: here and below clear destiny/interaction of all_ and recursive
# assert_equal(dsj['size']['total'],
# '15 Bytes' if (recursive and all_) else
# ('9 Bytes' if (recursive or all_) else '3 Bytes')
# )
# https://github.com/datalad/datalad/issues/1674
if state == 'file' and all_:
dirj = get_meta(topdir, 'dir')
dir_nodes = {x['name']: x for x in dirj['nodes']}
# it should be present in the subdir meta
assert_in('subds2', dir_nodes)
assert_not_in('url_external', dir_nodes['subds2'])
assert_in('subgit_ext', dir_nodes)
assert_equal(dir_nodes['subgit_ext']['url'], ext_url)
# and not in topds
assert_not_in('subds2', topds_nodes)
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
dsj = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
assert_equal(
topds_nodes['fromweb']['size']['total'], UNKNOWN_SIZE
)
def test_ls_json(topdir):
annex = AnnexRepo(topdir, create=True)
ds = Dataset(topdir)
# create some file and commit it
open(opj(ds.path, 'subdsfile.txt'), 'w').write('123')
ds.add(path='subdsfile.txt')
ds.save("Hello!", version_tag=1)
# add a subdataset
ds.install('subds', source=topdir)
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt'), commit=True) # commit to git to init git repo
annex.add(opj(topdir, 'dir', 'subgit'), commit=True) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir'), commit=True) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force']) # broken-link
meta_dir = opj('.git', 'datalad', 'metadata')
meta_path = opj(topdir, meta_dir)
def get_metahash(*path):
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
for all_ in [True, False]:
for recursive in [True, False]:
for state in ['file', 'delete']:
with swallow_logs(), swallow_outputs():
ds = _ls_json(topdir, json=state, all_=all_, recursive=recursive)
# subdataset should have its json created and deleted when all=True else not
subds_metahash = get_metahash('/')
subds_metapath = opj(topdir, 'subds', meta_dir, subds_metahash)
assert_equal(exists(subds_metapath), (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metahash = get_metahash('/')
ds_metapath = opj(meta_path, ds_metahash)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metahash = get_metahash('dir', 'subdir')
child_metapath = opj(meta_path, child_metahash)
assert_equal(exists(child_metapath), (state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden'), ('dir', 'subgit')]:
child_metahash = get_metahash(*subdir)
assert_equal(exists(opj(meta_path, child_metahash)), False)
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(ds['nodes'][0]['size']['total'], ds['size']['total'])
# check size of subdataset
subds = [item for item in ds['nodes'] if item['name'] == ('subdsfile.txt' or 'subds')][0]
assert_equal(subds['size']['total'], '3 Bytes')
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
ds = _ls_json(topdir, json='file', all_=False)
subds = [item for item in ds['nodes'] if item['name'] == ('subdsfile.txt' or 'subds')][0]
assert_equal(subds['size']['total'], '3 Bytes')
def test_fs_traverse(topdir):
# setup temp directory tree for testing
annex = AnnexRepo(topdir)
AnnexRepo(opj(topdir, 'annexdir'), create=True)
GitRepo(opj(topdir, 'gitdir'), create=True)
GitRepo(opj(topdir, 'dir', 'subgit'), create=True)
annex.add(opj(topdir, 'dir'), commit=True)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force'])
# traverse file system in recursive and non-recursive modes
for recursive in [True, False]:
# test fs_traverse in display mode
with swallow_logs(
new_level=logging.INFO) as log, swallow_outputs() as cmo:
fs = fs_traverse(topdir,
AnnexRepo(topdir),
recurse_directories=recursive,
json='display')
if recursive:
# fs_traverse logs should contain all not ignored subdirectories
for subdir in [
opj(topdir, 'dir'),
opj(topdir, 'dir', 'subdir')
]:
assert_in('Directory: ' + subdir, log.out)
# fs_traverse stdout contains subdirectory
assert_in(('file2.txt' and 'dir'), cmo.out)
# extract info of the top-level child directory
child = [item for item in fs['nodes'] if item['name'] == 'dir'][0]
# size of dir type child in non-recursive modes should be 0 Bytes(default) as
# dir type child's size currently has no metadata file for traverser to pick its size from
# and would require a recursive traversal w/ write to child metadata file mode
assert_equal(child['size']['total'], {
True: '6 Bytes',
False: '0 Bytes'
}[recursive])
for recursive in [True, False]:
# run fs_traverse in write to json 'file' mode
fs = fs_traverse(topdir,
AnnexRepo(topdir),
recurse_directories=recursive,
json='file')
# fs_traverse should return a dictionary
assert_equal(isinstance(fs, dict), True)
# not including git and annex folders
assert_equal([
item
for item in fs['nodes'] if ('gitdir' or 'annexdir') == item['name']
], [])
# extract info of the top-level child directory
child = [item for item in fs['nodes'] if item['name'] == 'dir'][0]
# verify node type
assert_equal(child['type'], 'dir')
# same node size on running fs_traversal in recursive followed by non-recursive mode
# verifies child's metadata file being used to find its size
# running in reverse order (non-recursive followed by recursive mode) will give (0, actual size)
assert_equal(child['size']['total'], '6 Bytes')
# verify subdirectory traversal if run in recursive mode
# In current RF 'nodes' are stripped away during recursive traversal
# for now... later we might reincarnate them "differently"
# TODO!
if False: # recursive:
# sub-dictionary should not include git and hidden directory info
assert_equal([
item for item in child['nodes']
if ('subgit' or '.fgit') == item['name']
], [])
# extract subdirectory dictionary, else fail
subchild = [
subitem for subitem in child["nodes"]
if subitem['name'] == 'subdir'
][0]
# extract info of file1.txts, else fail
link = [
subnode for subnode in subchild["nodes"]
if subnode['name'] == 'file1.txt'
][0]
# verify node's sizes and type
assert_equal(link['size']['total'], '3 Bytes')
assert_equal(link['size']['ondisk'], link['size']['total'])
assert_equal(link['type'], 'link')
# extract info of file2.txt, else fail
brokenlink = [
subnode for subnode in subchild["nodes"]
if subnode['name'] == 'file2.txt'
][0]
# verify node's sizes and type
assert_equal(brokenlink['type'], 'link-broken')
assert_equal(brokenlink['size']['ondisk'], '0 Bytes')
assert_equal(brokenlink['size']['total'], '3 Bytes')
def _test_AnnexDB(cls, path):
filepath1 = opj(path, 'file1.txt')
filep2 = opj('d', 'file2.txt')
filepath2 = opj(path, filep2)
annex = AnnexRepo(path, create=True)
# PhysicalFileStatusesDB relies on information in annex so files
# must be committed first
annex.add('file1.txt')
annex.commit("initial commit")
db = cls(annex=annex)
def set_db_status_from_file(fpath):
"""To test JsonFileStatusesDB, we need to keep updating the status stored"""
if cls is JsonFileStatusesDB:
# we need first to set the status
db.set(fpath, db._get_fileattributes_status(fpath))
set_db_status_from_file('file1.txt')
status1 = db.get('file1.txt')
assert(status1.size)
status1_ = db.get('file1.txt')
assert_equal(status1, status1_)
assert_false(db.is_different('file1.txt', status1))
assert_false(db.is_different('file1.txt', status1_))
# even if we add a filename specification
status1_.filename = 'file1.txt'
assert_false(db.is_different('file1.txt', status1_))
status1_.filename = 'different.txt'
assert_false(db.is_different('file1.txt', status1_))
os.unlink(filepath1) # under annex- - we don't have unlock yet and thus can't inplace augment
with open(filepath1, 'a') as f:
f.write('+')
# Note/TODO: fixed (realpath) path should go. Inner logic has to adapt to
# dataset singletons, that don't resolve symlinks
set_db_status_from_file(realpath(filepath1))
assert(db.is_different('file1.txt', status1))
# we should be able to get status of files out and inside of git
set_db_status_from_file('2git')
status_git1 = db.get('2git')
annex.add('2git', git=True)
annex.commit("added 2git")
assert_equal(db.get('2git'), status_git1)
# we should be able to get status of files with relative path to top dir and abs path
set_db_status_from_file(filep2)
status2 = db.get(filep2)
# Note/TODO: fixed (realpath) path should go. Inner logic has to adapt to
# dataset singletons, that don't resolve symlinks
status2_full = db.get(realpath(filepath2))
assert_equal(status2, status2_full)
# TODO? what about relative to curdir??
#with chpwd(opj(path, 'd')):
# status2_dir = db.get('./file2.txt')
# assert_equal(status2, status2_dir)
# since we asked about each file we added to DB/annex -- none should be
# known as "deleted"
assert_equal(db.get_obsolete(), [])
# Possibly save its state for persistent storage
#import pdb; pdb.set_trace()
db.save()
# but, if we create another DB which wasn't queried yet
db2 = cls(annex=annex)
# all files should be returned
# TODO: fixed by using realpath, but there should be a cleaner
# adaption to dataset singletons, that are NOT resolving symlinks, while the
# underlying repos do!
assert_equal(
set(db2.get_obsolete()),
{opj(realpath(path), p) for p in ['file1.txt', filep2, '2git']})
# and if we query one, it shouldn't be listed as deleted any more
status2_ = db2.get(filep2)
assert_equal(status2, status2_)
# TODO: fixed by using realpath, but there should be a cleaner
# adaption to dataset singletons, that are NOT resolving symlinks, while the
# underlying repos do!
assert_equal(
set(db2.get_obsolete()),
{opj(realpath(path), p) for p in ['file1.txt', '2git']})
# and if we queried with ./ prefix, should still work
db2.get(curdir + sep + 'file1.txt')
# TODO: fixed by using realpath, but there should be a cleaner
# adaption to dataset singletons, that are NOT resolving symlinks, while the
# underlying repos do!
assert_equal(
set(db2.get_obsolete()),
{opj(realpath(path), p) for p in ['2git']})
# and if we queried with a full path, should still work
# TODO: fixed by using realpath, but there should be a cleaner
# adaption to dataset singletons, that are NOT resolving symlinks, while the
# underlying repos do!
db2.get(opj(realpath(path), '2git'))
assert_equal(db2.get_obsolete(), [])
def test_ls_json(topdir, topurl):
annex = AnnexRepo(topdir, create=True)
ds = Dataset(topdir)
# create some file and commit it
with open(opj(ds.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
ds.save(path='subdsfile.txt', message="Hello!", version_tag=1)
# add a subdataset
ds.install('subds', source=topdir)
subdirds = ds.create(_path_('dir/subds2'), force=True)
subdirds.save('file')
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit',
'fgit.txt')) # commit to git to init git repo
git.commit()
annex.add(opj(topdir, 'dir',
'subgit')) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir')) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'),
options=['--force']) # broken-link
annex.commit()
git.add('fgit.txt') # commit to git to init git repo
git.commit()
# annex.add doesn't add submodule, so using ds.add
ds.save(opj('dir', 'subgit')) # add the non-dataset git repo to annex
ds.save('dir') # add to annex (links)
ds.drop(opj('dir', 'subdir', 'file2.txt'), check=False) # broken-link
# register "external" submodule by installing and uninstalling it
ext_url = topurl + '/dir/subgit/.git'
# need to make it installable via http
WitlessRunner(cwd=opj(topdir, 'dir', 'subgit')).run(
['git', 'update-server-info'])
ds.install(opj('dir', 'subgit_ext'), source=ext_url)
ds.uninstall(opj('dir', 'subgit_ext'))
meta_dir = opj('.git', 'datalad', 'metadata')
def get_metahash(*path):
if not path:
path = ['/']
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
def get_metapath(dspath, *path):
return _path_(dspath, meta_dir, get_metahash(*path))
def get_meta(dspath, *path):
with open(get_metapath(dspath, *path)) as f:
return js.load(f)
# Let's see that there is no crash if one of the files is available only
# in relaxed URL mode, so no size could be picked up
ds.repo.add_url_to_file('fromweb',
topurl + '/noteventhere',
options=['--relaxed'])
for all_ in [True, False]: # recurse directories
for recursive in [True, False]:
for state in ['file', 'delete']:
# subdataset should have its json created and deleted when
# all=True else not
subds_metapath = get_metapath(opj(topdir, 'subds'))
exists_prior = exists(subds_metapath)
#with swallow_logs(), swallow_outputs():
dsj = _ls_json(topdir,
json=state,
all_=all_,
recursive=recursive)
ok_startswith(dsj['tags'], '1-')
exists_post = exists(subds_metapath)
# print("%s %s -> %s" % (state, exists_prior, exists_post))
assert_equal(exists_post, (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metapath = get_metapath(topdir)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metapath = get_metapath(topdir, 'dir', 'subdir')
assert_equal(exists(child_metapath),
(state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden', ), ('dir', 'subgit')]:
assert_false(exists(get_metapath(topdir, *subdir)))
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(dsj['nodes'][0]['size']['total'],
dsj['size']['total'])
# check size of subdataset
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
# dir/subds2 must not be listed among nodes of the top dataset:
topds_nodes = {x['name']: x for x in dsj['nodes']}
assert_in('subds', topds_nodes)
# XXX
# # condition here is a bit a guesswork by yoh later on
# # TODO: here and below clear destiny/interaction of all_ and recursive
# assert_equal(dsj['size']['total'],
# '15 Bytes' if (recursive and all_) else
# ('9 Bytes' if (recursive or all_) else '3 Bytes')
# )
# https://github.com/datalad/datalad/issues/1674
if state == 'file' and all_:
dirj = get_meta(topdir, 'dir')
dir_nodes = {x['name']: x for x in dirj['nodes']}
# it should be present in the subdir meta
assert_in('subds2', dir_nodes)
assert_not_in('url_external', dir_nodes['subds2'])
assert_in('subgit_ext', dir_nodes)
assert_equal(dir_nodes['subgit_ext']['url'], ext_url)
# and not in topds
assert_not_in('subds2', topds_nodes)
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
dsj = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
assert_equal(topds_nodes['fromweb']['size']['total'],
UNKNOWN_SIZE)
def __call__(path=None,
force=False,
description=None,
dataset=None,
no_annex=False,
save=True,
annex_version=None,
annex_backend='MD5E',
native_metadata_type=None,
shared_access=None,
git_opts=None,
annex_opts=None,
annex_init_opts=None,
text_no_annex=None):
# two major cases
# 1. we got a `dataset` -> we either want to create it (path is None),
# or another dataset in it (path is not None)
# 2. we got no dataset -> we want to create a fresh dataset at the
# desired location, either at `path` or PWD
if path and dataset:
# Given a path and a dataset (path) not pointing to installed
# dataset
if not dataset.is_installed():
msg = "No installed dataset at %s found." % dataset.path
dsroot = get_dataset_root(dataset.path)
if dsroot:
msg += " If you meant to add to the %s dataset, use that path " \
"instead but remember that if dataset is provided, " \
"relative paths are relative to the top of the " \
"dataset." % dsroot
raise ValueError(msg)
# sanity check first
if git_opts:
lgr.warning(
"`git_opts` argument is presently ignored, please complain!")
if no_annex:
if description:
raise ValueError("Incompatible arguments: cannot specify "
"description for annex repo and declaring "
"no annex repo.")
if annex_opts:
raise ValueError("Incompatible arguments: cannot specify "
"options for annex and declaring no "
"annex repo.")
if annex_init_opts:
raise ValueError("Incompatible arguments: cannot specify "
"options for annex init and declaring no "
"annex repo.")
if not isinstance(force, bool):
raise ValueError(
"force should be bool, got %r. Did you mean to provide a 'path'?"
% force)
annotated_paths = AnnotatePaths.__call__(
# nothing given explicitly, assume create fresh right here
path=path if path else getpwd() if dataset is None else None,
dataset=dataset,
recursive=False,
action='create',
# we need to know whether we have to check for potential
# subdataset collision
force_parentds_discovery=True,
# it is absolutely OK to have something that does not exist
unavailable_path_status='',
unavailable_path_msg=None,
# if we have a dataset given that actually exists, we want to
# fail if the requested path is not in it
nondataset_path_status='error' \
if isinstance(dataset, Dataset) and dataset.is_installed() else '',
on_failure='ignore')
path = None
for r in annotated_paths:
if r['status']:
# this is dealt with already
yield r
continue
if path is not None:
raise ValueError(
"`create` can only handle single target path or dataset")
path = r
if len(annotated_paths) and path is None:
# we got something, we complained already, done
return
# we know that we need to create a dataset at `path`
assert (path is not None)
# prep for yield
path.update({'logger': lgr, 'type': 'dataset'})
# just discard, we have a new story to tell
path.pop('message', None)
if 'parentds' in path:
subs = Subdatasets.__call__(
dataset=path['parentds'],
# any known
fulfilled=None,
recursive=False,
contains=path['path'],
result_xfm='relpaths')
if len(subs):
path.update({
'status':
'error',
'message':
('collision with known subdataset %s/ in dataset %s',
subs[0], path['parentds'])
})
yield path
return
# TODO here we need a further test that if force=True, we need to look if
# there is a superdataset (regardless of whether we want to create a
# subdataset or not), and if that superdataset tracks anything within
# this directory -- if so, we need to stop right here and whine, because
# the result of creating a repo here will produce an undesired mess
if git_opts is None:
git_opts = {}
if shared_access:
# configure `git --shared` value
git_opts['shared'] = shared_access
# important to use the given Dataset object to avoid spurious ID
# changes with not-yet-materialized Datasets
tbds = dataset if isinstance(dataset, Dataset) and dataset.path == path['path'] \
else Dataset(path['path'])
# don't create in non-empty directory without `force`:
if isdir(tbds.path) and listdir(tbds.path) != [] and not force:
path.update({
'status':
'error',
'message':
'will not create a dataset in a non-empty directory, use '
'`force` option to ignore'
})
yield path
return
if no_annex:
lgr.info("Creating a new git repo at %s", tbds.path)
GitRepo(tbds.path, url=None, create=True, git_opts=git_opts)
else:
# always come with annex when created from scratch
lgr.info("Creating a new annex repo at %s", tbds.path)
tbrepo = AnnexRepo(tbds.path,
url=None,
create=True,
backend=annex_backend,
version=annex_version,
description=description,
git_opts=git_opts,
annex_opts=annex_opts,
annex_init_opts=annex_init_opts)
if text_no_annex:
git_attributes_file = opj(tbds.path, '.gitattributes')
with open(git_attributes_file, 'a') as f:
f.write('* annex.largefiles=(not(mimetype=text/*))\n')
tbrepo.add([git_attributes_file], git=True)
tbrepo.commit("Instructed annex to add text files to git",
_datalad_msg=True,
files=[git_attributes_file])
if native_metadata_type is not None:
if not isinstance(native_metadata_type, list):
native_metadata_type = [native_metadata_type]
for nt in native_metadata_type:
tbds.config.add('datalad.metadata.nativetype', nt)
# record an ID for this repo for the afterlife
# to be able to track siblings and children
id_var = 'datalad.dataset.id'
if id_var in tbds.config:
# make sure we reset this variable completely, in case of a re-create
tbds.config.unset(id_var, where='dataset')
tbds.config.add(id_var,
tbds.id if tbds.id is not None else
uuid.uuid1().urn.split(':')[-1],
where='dataset')
# make sure that v6 annex repos never commit content under .datalad
with open(opj(tbds.path, '.datalad', '.gitattributes'),
'a') as gitattr:
# TODO this will need adjusting, when annex'ed aggregate meta data
# comes around
gitattr.write(
'# Text files (according to file --mime-type) are added directly to git.\n'
)
gitattr.write(
'# See http://git-annex.branchable.com/tips/largefiles/ for more info.\n'
)
gitattr.write('** annex.largefiles=nothing\n')
# save everything, we need to do this now and cannot merge with the
# call below, because we may need to add this subdataset to a parent
# but cannot until we have a first commit
tbds.add('.datalad',
to_git=True,
save=save,
message='[DATALAD] new dataset')
# the next only makes sense if we saved the created dataset,
# otherwise we have no committed state to be registered
# in the parent
if save and isinstance(dataset, Dataset) and dataset.path != tbds.path:
# we created a dataset in another dataset
# -> make submodule
for r in dataset.add(tbds.path,
save=True,
return_type='generator',
result_filter=None,
result_xfm=None,
on_failure='ignore'):
yield r
path.update({'status': 'ok'})
yield path
def test_interactions(tdir):
# Just a placeholder since constructor expects a repo
repo = AnnexRepo(tdir, create=True, init=True)
repo.add('file.dat')
repo.commit('added file.dat')
class FIFO(object):
def __init__(self, content=None, default=None):
"""
Parameters
----------
content
default
If defined, will be the one returned if empty.
If not defined -- would raise an Exception
"""
self.content = content or []
self.default = default
def _pop(self):
# return empty line, usually to signal
if self.content:
v = self.content.pop(0)
# allow for debug
if v.startswith('DEBUG '):
# next one
return self._pop()
return v
else:
if self.default is not None:
return self.default
else:
raise IndexError("we are empty")
def write(self, l):
self.content.append(l)
def read(self):
return self._pop()
def readline(self):
return self._pop().rstrip('\n')
def flush(self):
pass # working hard
# now we should test interactions
import re
ERROR_ARGS = re.compile('^ERROR .*(missing|takes) .*\d+ .*argument')
for scenario in [
[], # default of doing nothing
[ # support of EXPORT which by default is not supported
('EXPORTSUPPORTED', 'EXPORTSUPPORTED-FAILURE'),
],
[ # some unknown option
('FANCYNEWOPTION', 'UNSUPPORTED-REQUEST'),
],
[
# get the COST etc for , and make sure we do not
# fail right on unsupported
('FANCYNEWOPTION', 'UNSUPPORTED-REQUEST'),
('GETCOST', 'COST %d' % DEFAULT_COST),
('GETCOST roguearg', ERROR_ARGS),
('GETAVAILABILITY', 'AVAILABILITY %s' % DEFAULT_AVAILABILITY),
('INITREMOTE', 'INITREMOTE-SUCCESS'
), # by default we do not require any fancy init
# no urls supported by default
('CLAIMURL http://example.com', 'CLAIMURL-FAILURE'),
# we know that is just a single option, url, is expected so full
# one would be passed
('CLAIMURL http://example.com roguearg', 'CLAIMURL-FAILURE'),
# but if not enough params -- ERROR_ARGS
('CLAIMURL', ERROR_ARGS)
]
]:
# First one is always version and
# Final empty command to signal the end of the transactions
scenario = [(None, 'VERSION 1')] + scenario + [('', None)]
fin, fout = FIFO(), FIFO(default='')
for in_, out_ in scenario:
if in_ is not None:
fin.write(in_ + '\n')
cr = AnnexCustomRemote(tdir, fin=fin, fout=fout)
cr.main()
for in_, out_ in scenario:
if out_ is not None:
out_read = fout.readline()
if isinstance(out_, type(ERROR_ARGS)):
assert out_.match(out_read), (out_, out_read)
else:
eq_(out_, out_read)
out_read = fout.readline()
eq_(out_read, '') # nothing left to say
