def test_install_subds_from_another_remote(topdir):
# https://github.com/datalad/datalad/issues/1905
from datalad.support.network import PathRI
with chpwd(topdir):
origin_ = 'origin'
clone1_ = 'clone1'
clone2_ = 'clone2'
origin = create(origin_, no_annex=True)
clone1 = install(source=origin, path=clone1_)
# print("Initial clone")
clone1.create_sibling('ssh://localhost%s/%s' %
(PathRI(getpwd()).posixpath, clone2_),
name=clone2_)
# print("Creating clone2")
clone1.publish(to=clone2_)
clone2 = Dataset(clone2_)
# print("Initiating subdataset")
clone2.create('subds1')
# print("Updating")
clone1.update(merge=True, sibling=clone2_)
# print("Installing within updated dataset -- should be able to install from clone2")
clone1.install('subds1')
def test_url_eq():
eq_(URL(), URL())
# doesn't make sense to ask what kind of a url it is an empty URL
#eq_(RI(), RI())
neq_(URL(), URL(hostname='x'))
# Different types aren't equal even if have the same fields values
neq_(URL(path='x'), PathRI(path='x'))
neq_(URL(hostname='x'), SSHRI(hostname='x'))
neq_(str(URL(hostname='x')), str(SSHRI(hostname='x')))
def test_smth_about_not_supported(p1, p2):
source = Dataset(p1).create()
from datalad.support.network import PathRI
source.create_sibling('ssh://localhost' + PathRI(p2).posixpath,
name='target1')
# source.publish(to='target1')
with chpwd(p1):
# since we have only two commits (set backend, init dataset)
# -- there is no HEAD^^
assert_result_count(publish(to='target1',
since='HEAD^^',
on_failure='ignore'),
1,
status='impossible',
message="fatal: bad revision 'HEAD^^'")
# but now let's add one more commit, we should be able to pusblish
source.repo.commit("msg", options=['--allow-empty'])
publish(to='target1', since='HEAD^') # must not fail now
def __call__(path,
dataset=None,
spec_file=None,
properties=None,
replace=False):
# TODO: message
dataset = require_dataset(dataset,
check_installed=True,
purpose="hirni spec4anything")
path = assure_list(path)
path = [resolve_path(p, dataset) for p in path]
res_kwargs = dict(action='hirni spec4anything', logger=lgr)
res_kwargs['refds'] = Interface.get_refds_path(dataset)
# ### This might become superfluous. See datalad-gh-2653
ds_path = PathRI(dataset.path)
# ###
updated_files = []
paths = []
for ap in AnnotatePaths.__call__(
dataset=dataset,
path=path,
action='hirni spec4anything',
unavailable_path_status='impossible',
nondataset_path_status='error',
return_type='generator',
# TODO: Check this one out:
on_failure='ignore',
# Note/TODO: Not sure yet whether and when we need those.
# Generally we want to be able to create a spec for subdatasets,
# too:
# recursive=recursive,
# recursion_limit=recursion_limit,
# force_subds_discovery=True,
# force_parentds_discovery=True,
):
if ap.get('status', None) in ['error', 'impossible']:
yield ap
continue
# ### This might become superfluous. See datalad-gh-2653
ap_path = PathRI(ap['path'])
# ###
# find acquisition and respective specification file:
rel_path = posixpath.relpath(ap_path.posixpath, ds_path.posixpath)
path_parts = rel_path.split('/')
# TODO: Note: Outcommented this warning for now. We used to not have
# a spec file at the toplevel of the study dataset, but now we do.
# The logic afterwards works, but should be revisited. At least,
# `acq` should be called differently now.
# if len(path_parts) < 2:
# lgr.warning("Not within an acquisition")
acq = path_parts[0]
# TODO: spec file specifiable or fixed path?
# if we want the former, what we actually need is an
# association of acquisition and its spec path
# => prob. not an option but a config
spec_path = spec_file if spec_file \
else posixpath.join(ds_path.posixpath, acq,
dataset.config.get("datalad.hirni.studyspec.filename",
"studyspec.json"))
spec = [r for r in json_py.load_stream(spec_path)] \
if posixpath.exists(spec_path) else list()
lgr.debug("Add specification snippet for %s", ap['path'])
# XXX 'add' does not seem to be the thing we want to do
# rather 'set', so we have to check whether a spec for a location
# is already known and fail or replace it (maybe with --force)
# go through all existing specs and extract unique value
# and also assign them to the new record (subjects, ...), but only
# editable fields!!
uniques = dict()
for s in spec:
for k in s:
if isinstance(s[k], dict) and 'value' in s[k]:
if k not in uniques:
uniques[k] = set()
uniques[k].add(s[k]['value'])
overrides = dict()
for k in uniques:
if len(uniques[k]) == 1:
overrides[k] = _get_edit_dict(value=uniques[k].pop(),
approved=False)
if properties:
# TODO: This entire reading of properties needs to be RF'd
# into proper generalized functions.
# spec got more complex. update() prob. can't simply override
# (think: 'procedures' and 'tags' prob. need to be appended
# instead)
# load from file or json string
if isinstance(properties, dict):
props = properties
elif op.exists(properties):
props = json_py.load(properties)
else:
props = json_py.loads(properties)
# turn into editable, pre-approved records
spec_props = {
k: dict(value=v, approved=True)
for k, v in props.items()
if k not in non_editables + ['tags', 'procedures']
}
spec_props.update({
k: v
for k, v in props.items() if k in non_editables + ['tags']
})
# TODO: still wrong. It's a list. Append or override? How to decide?
spec_props.update({
o_k: [{
i_k: dict(value=i_v, approved=True)
for i_k, i_v in o_v.items()
}]
for o_k, o_v in props.items() if o_k in ['procedures']
})
overrides.update(spec_props)
# TODO: It's probably wrong to use uniques for overwriting! At least
# they cannot be used to overwrite values explicitly set in
# _add_to_spec like "location", "type", etc.
#
# But then: This should concern non-editable fields only, right?
spec = _add_to_spec(spec,
posixpath.split(spec_path)[0],
ap,
dataset,
overrides=overrides,
replace=replace)
# Note: Not sure whether we really want one commit per snippet.
# If not - consider:
# - What if we fail amidst? => Don't write to file yet.
# - What about input paths from different acquisitions?
# => store specs per acquisition in memory
# MIH: One commit per line seems silly. why not update all files
# collect paths of updated files, and give them to a single `add`
# at the very end?
# MIH: if we fail, we fail and nothing is committed
from datalad_hirni.support.spec_helpers import sort_spec
json_py.dump2stream(sorted(spec, key=lambda x: sort_spec(x)),
spec_path)
updated_files.append(spec_path)
yield get_status_dict(status='ok',
type=ap['type'],
path=ap['path'],
**res_kwargs)
paths.append(ap)
from datalad.dochelpers import single_or_plural
from os import linesep
message = "[HIRNI] Add specification {n_snippets} for: {paths}".format(
n_snippets=single_or_plural("snippet", "snippets", len(paths)),
paths=linesep.join(" - " + op.relpath(p['path'], dataset.path)
for p in paths)
if len(paths) > 1 else op.relpath(paths[0]['path'], dataset.path))
for r in dataset.save(updated_files,
to_git=True,
message=message,
return_type='generator',
result_renderer='disabled'):
yield r