def __call__(loc,
recursive=False,
fast=False,
all_=False,
long_=False,
config_file=None,
list_content=False,
json=None):
if json:
from datalad.interface.ls_webui import _ls_json
if isinstance(loc, list) and not len(loc):
# nothing given, CWD assumed -- just like regular ls
loc = '.'
kw = dict(fast=fast, recursive=recursive, all_=all_, long_=long_)
if isinstance(loc, list):
return [
Ls.__call__(loc_,
config_file=config_file,
list_content=list_content,
json=json,
**kw) for loc_ in loc
]
# TODO: do some clever handling of kwargs as to remember what were defaults
# and what any particular implementation actually needs, and then issuing
# warning if some custom value/option was specified which doesn't apply to the
# given url
# rename to not angry Python gods who took all_ good words
kw['long_'] = kw.pop('long_')
loc_type = "unknown"
if loc.startswith('s3://'):
return _ls_s3(loc,
config_file=config_file,
list_content=list_content,
**kw)
elif lexists(loc):
if isdir(loc):
ds = Dataset(loc)
if ds.is_installed():
return _ls_json(loc, json=json, **
kw) if json else _ls_dataset(loc, **kw)
loc_type = False
else:
loc_type = "dir" # we know that so far for sure
# it might have been an uninstalled dataset within super-dataset
superds = ds.get_superdataset()
if superds:
try:
subdatasets = Ls._cached_subdatasets[superds.path]
except KeyError:
subdatasets = Ls._cached_subdatasets[superds.path] \
= superds.subdatasets(result_xfm='relpaths')
if relpath(ds.path, superds.path) in subdatasets:
loc_type = "not installed"
else:
loc_type = "file"
# could list properties -- under annex or git, either clean/dirty
# etc
# repo = get_repo_instance(dirname(loc))
if loc_type:
#raise ValueError("ATM supporting only s3:// URLs and paths to local datasets")
# TODO: unify all_ the output here -- _ls functions should just return something
# to be displayed
ui.message("{} {}".format(
ansi_colors.color_word(loc, ansi_colors.DATASET),
ansi_colors.color_word(
loc_type, ansi_colors.RED if loc_type
in {'unknown', 'not installed'} else ansi_colors.BLUE)))
def test_ls_json(topdir):
annex = AnnexRepo(topdir, create=True)
dsj = Dataset(topdir)
# create some file and commit it
with open(opj(dsj.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
dsj.add(path='subdsfile.txt')
dsj.save("Hello!", version_tag=1)
# add a subdataset
dsj.install('subds', source=topdir)
subdirds = dsj.create(_path_('dir/subds2'), force=True)
subdirds.add('file')
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt'),
commit=True) # commit to git to init git repo
annex.add(opj(topdir, 'dir', 'subgit'),
commit=True) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir'), commit=True) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'),
options=['--force']) # broken-link
meta_dir = opj('.git', 'datalad', 'metadata')
meta_path = opj(topdir, meta_dir)
def get_metahash(*path):
if not path:
path = ['/']
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
def get_metapath(dspath, *path):
return _path_(dspath, meta_dir, get_metahash(*path))
def get_meta(dspath, *path):
with open(get_metapath(dspath, *path)) as f:
return js.load(f)
for all_ in [True, False]: # recurse directories
for recursive in [True, False]:
for state in ['file', 'delete']:
# subdataset should have its json created and deleted when
# all=True else not
subds_metapath = get_metapath(opj(topdir, 'subds'))
exists_prior = exists(subds_metapath)
#with swallow_logs(), swallow_outputs():
dsj = _ls_json(topdir,
json=state,
all_=all_,
recursive=recursive)
exists_post = exists(subds_metapath)
# print("%s %s -> %s" % (state, exists_prior, exists_post))
assert_equal(exists_post, (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metapath = get_metapath(topdir)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metapath = get_metapath(topdir, 'dir', 'subdir')
assert_equal(exists(child_metapath),
(state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden', ), ('dir', 'subgit')]:
assert_false(exists(get_metapath(topdir, *subdir)))
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(dsj['nodes'][0]['size']['total'],
dsj['size']['total'])
# check size of subdataset
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
# dir/subds2 must not be listed among nodes of the top dataset:
topds_nodes = {x['name']: x for x in dsj['nodes']}
assert_in('subds', topds_nodes)
# XXX
# # condition here is a bit a guesswork by yoh later on
# # TODO: here and below clear destiny/interaction of all_ and recursive
# assert_equal(dsj['size']['total'],
# '15 Bytes' if (recursive and all_) else
# ('9 Bytes' if (recursive or all_) else '3 Bytes')
# )
# https://github.com/datalad/datalad/issues/1674
if state == 'file' and all_:
dirj = get_meta(topdir, 'dir')
dir_nodes = {x['name']: x for x in dirj['nodes']}
# it should be present in the subdir meta
assert_in('subds2', dir_nodes)
# and not in topds
assert_not_in('subds2', topds_nodes)
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
dsj = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
def test_ls_json(topdir, topurl):
annex = AnnexRepo(topdir, create=True)
ds = Dataset(topdir)
# create some file and commit it
with open(opj(ds.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
ds.save(path='subdsfile.txt', message="Hello!", version_tag=1)
# add a subdataset
ds.install('subds', source=topdir)
subdirds = ds.create(_path_('dir/subds2'), force=True)
subdirds.save('file')
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit',
'fgit.txt')) # commit to git to init git repo
git.commit()
annex.add(opj(topdir, 'dir',
'subgit')) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir')) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'),
options=['--force']) # broken-link
annex.commit()
git.add('fgit.txt') # commit to git to init git repo
git.commit()
# annex.add doesn't add submodule, so using ds.add
ds.save(opj('dir', 'subgit')) # add the non-dataset git repo to annex
ds.save('dir') # add to annex (links)
ds.drop(opj('dir', 'subdir', 'file2.txt'), check=False) # broken-link
# register "external" submodule by installing and uninstalling it
ext_url = topurl + '/dir/subgit/.git'
# need to make it installable via http
WitlessRunner(cwd=opj(topdir, 'dir', 'subgit')).run(
['git', 'update-server-info'])
ds.install(opj('dir', 'subgit_ext'), source=ext_url)
ds.uninstall(opj('dir', 'subgit_ext'))
meta_dir = opj('.git', 'datalad', 'metadata')
def get_metahash(*path):
if not path:
path = ['/']
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
def get_metapath(dspath, *path):
return _path_(dspath, meta_dir, get_metahash(*path))
def get_meta(dspath, *path):
with open(get_metapath(dspath, *path)) as f:
return js.load(f)
# Let's see that there is no crash if one of the files is available only
# in relaxed URL mode, so no size could be picked up
ds.repo.add_url_to_file('fromweb',
topurl + '/noteventhere',
options=['--relaxed'])
for all_ in [True, False]: # recurse directories
for recursive in [True, False]:
for state in ['file', 'delete']:
# subdataset should have its json created and deleted when
# all=True else not
subds_metapath = get_metapath(opj(topdir, 'subds'))
exists_prior = exists(subds_metapath)
#with swallow_logs(), swallow_outputs():
dsj = _ls_json(topdir,
json=state,
all_=all_,
recursive=recursive)
ok_startswith(dsj['tags'], '1-')
exists_post = exists(subds_metapath)
# print("%s %s -> %s" % (state, exists_prior, exists_post))
assert_equal(exists_post, (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metapath = get_metapath(topdir)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metapath = get_metapath(topdir, 'dir', 'subdir')
assert_equal(exists(child_metapath),
(state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden', ), ('dir', 'subgit')]:
assert_false(exists(get_metapath(topdir, *subdir)))
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(dsj['nodes'][0]['size']['total'],
dsj['size']['total'])
# check size of subdataset
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
# dir/subds2 must not be listed among nodes of the top dataset:
topds_nodes = {x['name']: x for x in dsj['nodes']}
assert_in('subds', topds_nodes)
# XXX
# # condition here is a bit a guesswork by yoh later on
# # TODO: here and below clear destiny/interaction of all_ and recursive
# assert_equal(dsj['size']['total'],
# '15 Bytes' if (recursive and all_) else
# ('9 Bytes' if (recursive or all_) else '3 Bytes')
# )
# https://github.com/datalad/datalad/issues/1674
if state == 'file' and all_:
dirj = get_meta(topdir, 'dir')
dir_nodes = {x['name']: x for x in dirj['nodes']}
# it should be present in the subdir meta
assert_in('subds2', dir_nodes)
assert_not_in('url_external', dir_nodes['subds2'])
assert_in('subgit_ext', dir_nodes)
assert_equal(dir_nodes['subgit_ext']['url'], ext_url)
# and not in topds
assert_not_in('subds2', topds_nodes)
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
dsj = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
assert_equal(topds_nodes['fromweb']['size']['total'],
UNKNOWN_SIZE)
def __call__(loc, recursive=False, fast=False, all_=False, long_=False,
config_file=None, list_content=False, json=None):
if json:
from datalad.interface.ls_webui import _ls_json
if isinstance(loc, list) and not len(loc):
# nothing given, CWD assumed -- just like regular ls
loc = '.'
kw = dict(fast=fast, recursive=recursive, all_=all_, long_=long_)
if isinstance(loc, list):
return [Ls.__call__(loc_, config_file=config_file,
list_content=list_content, json=json, **kw)
for loc_ in loc]
# TODO: do some clever handling of kwargs as to remember what were defaults
# and what any particular implementation actually needs, and then issuing
# warning if some custom value/option was specified which doesn't apply to the
# given url
# rename to not angry Python gods who took all_ good words
kw['long_'] = kw.pop('long_')
loc_type = "unknown"
if loc.startswith('s3://'):
return _ls_s3(loc, config_file=config_file, list_content=list_content,
**kw)
elif lexists(loc):
if isdir(loc):
ds = Dataset(loc)
if ds.is_installed():
return _ls_json(loc, json=json, **kw) if json else _ls_dataset(loc, **kw)
loc_type = False
else:
loc_type = "dir" # we know that so far for sure
# it might have been an uninstalled dataset within super-dataset
superds = ds.get_superdataset()
if superds:
try:
subdatasets = Ls._cached_subdatasets[superds.path]
except KeyError:
subdatasets = Ls._cached_subdatasets[superds.path] \
= superds.subdatasets(result_xfm='relpaths')
if relpath(ds.path, superds.path) in subdatasets:
loc_type = "not installed"
else:
loc_type = "file"
# could list properties -- under annex or git, either clean/dirty
# etc
# repo = get_repo_instance(dirname(loc))
if loc_type:
#raise ValueError("ATM supporting only s3:// URLs and paths to local datasets")
# TODO: unify all_ the output here -- _ls functions should just return something
# to be displayed
ui.message(
"{} {}".format(
ansi_colors.color_word(loc, ansi_colors.DATASET),
ansi_colors.color_word(
loc_type,
ansi_colors.RED
if loc_type in {'unknown', 'not installed'}
else ansi_colors.BLUE)
)
)
def test_ls_json(topdir, topurl):
annex = AnnexRepo(topdir, create=True)
ds = Dataset(topdir)
# create some file and commit it
with open(opj(ds.path, 'subdsfile.txt'), 'w') as f:
f.write('123')
ds.add(path='subdsfile.txt')
ds.save("Hello!", version_tag=1)
# add a subdataset
ds.install('subds', source=topdir)
subdirds = ds.create(_path_('dir/subds2'), force=True)
subdirds.add('file')
git = GitRepo(opj(topdir, 'dir', 'subgit'), create=True) # create git repo
git.add(opj(topdir, 'dir', 'subgit', 'fgit.txt')) # commit to git to init git repo
git.commit()
annex.add(opj(topdir, 'dir', 'subgit')) # add the non-dataset git repo to annex
annex.add(opj(topdir, 'dir')) # add to annex (links)
annex.drop(opj(topdir, 'dir', 'subdir', 'file2.txt'), options=['--force']) # broken-link
annex.commit()
git.add('fgit.txt') # commit to git to init git repo
git.commit()
# annex.add doesn't add submodule, so using ds.add
ds.add(opj('dir', 'subgit')) # add the non-dataset git repo to annex
ds.add('dir') # add to annex (links)
ds.drop(opj('dir', 'subdir', 'file2.txt'), check=False) # broken-link
# register "external" submodule by installing and uninstalling it
ext_url = topurl + '/dir/subgit/.git'
# need to make it installable via http
Runner()('git update-server-info', cwd=opj(topdir, 'dir', 'subgit'))
ds.install(opj('dir', 'subgit_ext'), source=ext_url)
ds.uninstall(opj('dir', 'subgit_ext'))
meta_dir = opj('.git', 'datalad', 'metadata')
def get_metahash(*path):
if not path:
path = ['/']
return hashlib.md5(opj(*path).encode('utf-8')).hexdigest()
def get_metapath(dspath, *path):
return _path_(dspath, meta_dir, get_metahash(*path))
def get_meta(dspath, *path):
with open(get_metapath(dspath, *path)) as f:
return js.load(f)
# Let's see that there is no crash if one of the files is available only
# in relaxed URL mode, so no size could be picked up
ds.repo.add_url_to_file(
'fromweb', topurl + '/noteventhere', options=['--relaxed'])
for all_ in [True, False]: # recurse directories
for recursive in [True, False]:
for state in ['file', 'delete']:
# subdataset should have its json created and deleted when
# all=True else not
subds_metapath = get_metapath(opj(topdir, 'subds'))
exists_prior = exists(subds_metapath)
#with swallow_logs(), swallow_outputs():
dsj = _ls_json(
topdir,
json=state,
all_=all_,
recursive=recursive
)
ok_startswith(dsj['tags'], '1-')
exists_post = exists(subds_metapath)
# print("%s %s -> %s" % (state, exists_prior, exists_post))
assert_equal(exists_post, (state == 'file' and recursive))
# root should have its json file created and deleted in all cases
ds_metapath = get_metapath(topdir)
assert_equal(exists(ds_metapath), state == 'file')
# children should have their metadata json's created and deleted only when recursive=True
child_metapath = get_metapath(topdir, 'dir', 'subdir')
assert_equal(exists(child_metapath), (state == 'file' and all_))
# ignored directories should not have json files created in any case
for subdir in [('.hidden',), ('dir', 'subgit')]:
assert_false(exists(get_metapath(topdir, *subdir)))
# check if its updated in its nodes sublist too. used by web-ui json. regression test
assert_equal(dsj['nodes'][0]['size']['total'], dsj['size']['total'])
# check size of subdataset
subds = [item for item in dsj['nodes'] if item['name'] == ('subdsfile.txt' or 'subds')][0]
assert_equal(subds['size']['total'], '3 Bytes')
# dir/subds2 must not be listed among nodes of the top dataset:
topds_nodes = {x['name']: x for x in dsj['nodes']}
assert_in('subds', topds_nodes)
# XXX
# # condition here is a bit a guesswork by yoh later on
# # TODO: here and below clear destiny/interaction of all_ and recursive
# assert_equal(dsj['size']['total'],
# '15 Bytes' if (recursive and all_) else
# ('9 Bytes' if (recursive or all_) else '3 Bytes')
# )
# https://github.com/datalad/datalad/issues/1674
if state == 'file' and all_:
dirj = get_meta(topdir, 'dir')
dir_nodes = {x['name']: x for x in dirj['nodes']}
# it should be present in the subdir meta
assert_in('subds2', dir_nodes)
assert_not_in('url_external', dir_nodes['subds2'])
assert_in('subgit_ext', dir_nodes)
assert_equal(dir_nodes['subgit_ext']['url'], ext_url)
# and not in topds
assert_not_in('subds2', topds_nodes)
# run non-recursive dataset traversal after subdataset metadata already created
# to verify sub-dataset metadata being picked up from its metadata file in such cases
if state == 'file' and recursive and not all_:
dsj = _ls_json(topdir, json='file', all_=False)
subds = [
item for item in dsj['nodes']
if item['name'] == ('subdsfile.txt' or 'subds')
][0]
assert_equal(subds['size']['total'], '3 Bytes')
assert_equal(
topds_nodes['fromweb']['size']['total'], UNKNOWN_SIZE
)