def _get_targetpaths(ds, content, refds_path, source, jobs):
# not ready for Path instances...
content = [str(c) for c in content]
# hand over to git-annex, get files content,
# report files in git as 'notneeded' to get
ds_repo = ds.repo
# needs to be an annex to get content
if not isinstance(ds_repo, AnnexRepo):
for r in results_from_paths(
content,
status='notneeded',
message="no dataset annex, content already present",
action='get',
type='file',
logger=lgr,
refds=refds_path):
yield r
return
respath_by_status = {}
try:
results = ds_repo.get(content,
options=['--from=%s' % source] if source else [],
jobs=jobs)
except CommandError as exc:
results = exc.kwargs.get("stdout_json")
if not results:
raise
for res in results:
res = annexjson2result(res,
ds,
type='file',
logger=lgr,
refds=refds_path)
success = success_status_map[res['status']]
# TODO: in case of some failed commands (e.g. get) there might
# be no path in the record. yoh has only vague idea of logic
# here so just checks for having 'path', but according to
# results_from_annex_noinfo, then it would be assumed that
# `content` was acquired successfully, which is not the case
if 'path' in res:
respath_by_status[success] = \
respath_by_status.get(success, []) + [res['path']]
yield res
for r in results_from_annex_noinfo(
ds,
content,
respath_by_status,
dir_fail_msg='could not get some content in %s %s',
noinfo_dir_msg='nothing to get from %s',
noinfo_file_msg='already present',
action='get',
logger=lgr,
refds=refds_path):
yield r
def __call__(
path=None,
source=None,
dataset=None,
recursive=False,
recursion_limit=None,
get_data=True,
description=None,
reckless=False,
#git_opts=None,
#annex_opts=None,
#annex_get_opts=None,
jobs='auto',
verbose=False,
):
# IMPLEMENTATION CONCEPT:
#
# 1. Sort the world into existing handles and the rest
# 2. Try locate missing handles (obtain subdatasets along the way)
# 3. Expand into subdatasets with recursion enables (potentially
# obtain even more subdatasets
# 4. Shoot info of which handles to get in each subdataset to,
# git-annex, once at the very end
refds_path = Interface.get_refds_path(dataset)
if not (dataset or path):
raise InsufficientArgumentsError(
"Neither dataset nor target path(s) provided")
if dataset and not path:
# act on the whole dataset if nothing else was specified
path = refds_path
# remember which results we already reported, to avoid duplicates
yielded_ds = []
to_get = []
unavailable_paths = []
for ap in AnnotatePaths.__call__(
path=path,
dataset=refds_path,
recursive=recursive,
recursion_limit=recursion_limit,
action='get',
# NOTE: Do not act upon unavailable paths yet! Done below after
# testing which ones could be obtained
unavailable_path_status='',
nondataset_path_status='impossible',
return_type='generator',
on_failure='ignore'):
if ap.get('status', None):
# we know what to report already
yield ap
continue
if ap.get('state', None) == 'absent' and ap.get(
'raw_input', False):
# if this wasn't found, but directly requested, queue for further
# exploration
unavailable_paths.append(ap)
continue
if ap.get('type', None) == 'dataset' and \
GitRepo.is_valid_repo(ap['path']) and \
not ap['path'] == refds_path:
# do not report what hasn't arived yet
# also do not report the base dataset that is already
# present -- no surprise
yield dict(ap,
status='notneeded',
logger=lgr,
message='already installed')
yielded_ds.append(ap['path'])
ap['process_content'] = get_data
to_get.append(ap)
# explore the unknown
for ap in sorted(unavailable_paths, key=lambda x: x['path']):
lgr.debug("Investigate yet unavailable path %s", ap)
# how close can we get?
dspath = ap.get('parentds', get_dataset_root(ap['path']))
if dspath is None:
# nothing we can do for this path
continue
lgr.debug("Found containing dataset %s for path %s", dspath,
ap['path'])
ds = Dataset(dspath)
# now actually obtain whatever is necessary to get to this path
containing_ds = [dspath]
for res in _install_necessary_subdatasets(ds,
ap['path'],
reckless,
refds_path,
description=description):
# yield immediately so errors could be acted upon outside, before
# we continue
if not (res['type'] == 'dataset'
and res['path'] in yielded_ds):
# unless we reported on this dataset before
if res['type'] == 'dataset':
# make a record, recursive below might now want to report
# a 'notneeded'
yielded_ds.append(res['path'])
yield res
# update to the current innermost dataset
containing_ds.append(res['path'])
if len(containing_ds) < 2:
# no subdataset was installed, hence if the path was unavailable
# before it still is, no need to bother git annex
ap.update(status='impossible', message='path does not exist')
yield ap
continue
# important to only do the next for the innermost subdataset
# as the `recursive` logic below relies on that!
# set the correct parent, for a dataset this would be the second-last
# reported subdataset
ap.update(parentds=containing_ds[-1])
if containing_ds[-1] == ap['path']:
# the path actually refers to the last installed dataset
ap.update(parentds=containing_ds[-2],
process_content=get_data,
type='dataset')
to_get.append(ap)
# results of recursive installation of yet undiscovered datasets
rec_get = []
if recursive and not recursion_limit == 'existing':
# obtain any subdatasets underneath the paths given inside the
# subdatasets that we know already exist
# unless we do not want recursion into not-yet-installed datasets
for ap in sorted(to_get, key=lambda x: x['path']):
if ap['type'] not in ('dataset', 'directory') or not ap.get(
'raw_input', False):
# a non-directory cannot have content underneath
# also we do NOT want to recurse into anything that was specifically
# requested, to avoid duplication
continue
subds = Dataset(ap['path'] if ap['type'] ==
'dataset' else ap['parentds'])
lgr.info("Installing %s%s recursively", subds,
(" underneath %s" %
ap['path'] if subds.path != ap['path'] else ""))
for res in _recursive_install_subds_underneath(
subds,
# `ap['path']` was explicitly given as input
# we count recursions from the input, hence we
# can start with the full number
recursion_limit,
reckless,
start=ap['path'],
refds_path=refds_path,
description=description):
# yield immediately so errors could be acted upon
# outside, before we continue
if not (res['type'] == 'dataset'
and res['path'] in yielded_ds):
# unless we reported on this dataset before
if res['type'] == 'dataset':
# make a record
yielded_ds.append(res['path'])
yield res
if not (res['status'] == 'ok'
and res['type'] == 'dataset'):
# not a dataset that was just installed, we just reported it
# upstairs, and can ignore it from now on
continue
# paranoia, so popular these days...
assert GitRepo.is_valid_repo(res['path'])
# keep a copy of the install record for `get` later on
get_ap = {
k: v
for k, v in res.items() if not k == 'status'
}
get_ap['process_content'] = get_data
rec_get.append(get_ap)
if not get_data:
# done already
return
# merge the two AP lists
to_get.extend(rec_get)
# sort into datasets
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
to_get,
refds_path=refds_path)
assert (not completed)
# hand over to git-annex, get files content,
# report files in git as 'notneeded' to get
for ds_path in sorted(content_by_ds.keys()):
ds = Dataset(ds_path)
# grab content, ignore subdataset entries
content = [
ap['path'] for ap in content_by_ds[ds_path]
if ap.get('type', None) != 'dataset' or ap['path'] == ds.path
]
if not content:
# cut this short should there be nothing
continue
# needs to be an annex to get content
if not isinstance(ds.repo, AnnexRepo):
for r in results_from_paths(
content,
status='notneeded',
message="no dataset annex, content already present",
action='get',
logger=lgr,
refds=refds_path):
yield r
continue
respath_by_status = {}
for res in ds.repo.get(content,
options=['--from=%s' %
source] if source else [],
jobs=jobs):
res = annexjson2result(res,
ds,
type='file',
logger=lgr,
refds=refds_path)
success = success_status_map[res['status']]
# TODO: in case of some failed commands (e.g. get) there might
# be no path in the record. yoh has only vague idea of logic
# here so just checks for having 'path', but according to
# results_from_annex_noinfo, then it would be assumed that
# `content` was acquired successfully, which is not the case
if 'path' in res:
respath_by_status[success] = \
respath_by_status.get(success, []) + [res['path']]
yield res
for r in results_from_annex_noinfo(
ds,
content,
respath_by_status,
dir_fail_msg='could not get some content in %s %s',
noinfo_dir_msg='nothing to get from %s',
noinfo_file_msg='already present',
action='get',
logger=lgr,
refds=refds_path):
yield r
def __call__(
path=None,
source=None,
dataset=None,
recursive=False,
recursion_limit=None,
get_data=True,
description=None,
reckless=False,
#git_opts=None,
#annex_opts=None,
#annex_get_opts=None,
jobs='auto',
verbose=False,
):
# IMPLEMENTATION CONCEPT:
#
# 1. Sort the world into existing handles and the rest
# 2. Try locate missing handles (obtain subdatasets along the way)
# 3. Expand into subdatasets with recursion enables (potentially
# obtain even more subdatasets
# 4. Shoot info of which handles to get in each subdataset to,
# git-annex, once at the very end
refds_path = Interface.get_refds_path(dataset)
if not (dataset or path):
raise InsufficientArgumentsError(
"Neither dataset nor target path(s) provided")
if dataset and not path:
# act on the whole dataset if nothing else was specified
path = refds_path
# remember which results we already reported, to avoid duplicates
yielded_ds = []
to_get = []
unavailable_paths = []
for ap in AnnotatePaths.__call__(
path=path,
dataset=refds_path,
recursive=recursive,
recursion_limit=recursion_limit,
action='get',
# NOTE: Do not act upon unavailable paths yet! Done below after
# testing which ones could be obtained
unavailable_path_status='',
nondataset_path_status='impossible',
return_type='generator',
on_failure='ignore'):
if ap.get('status', None):
# we know what to report already
yield ap
continue
if ap.get('state', None) == 'absent' and ap.get('raw_input', False):
# if this wasn't found, but directly requested, queue for further
# exploration
unavailable_paths.append(ap)
continue
if ap.get('type', None) == 'dataset' and \
GitRepo.is_valid_repo(ap['path']) and \
not ap['path'] == refds_path:
# do not report what hasn't arived yet
# also do not report the base dataset that is already
# present -- no surprise
yield dict(ap, status='notneeded', logger=lgr,
message='already installed')
yielded_ds.append(ap['path'])
ap['process_content'] = get_data
to_get.append(ap)
# explore the unknown
for ap in sorted(unavailable_paths, key=lambda x: x['path']):
lgr.debug("Investigate yet unavailable path %s", ap)
# how close can we get?
dspath = ap.get('parentds', get_dataset_root(ap['path']))
if dspath is None:
# nothing we can do for this path
continue
lgr.debug("Found containing dataset %s for path %s", dspath, ap['path'])
ds = Dataset(dspath)
# now actually obtain whatever is necessary to get to this path
containing_ds = [dspath]
for res in _install_necessary_subdatasets(
ds, ap['path'], reckless, refds_path, description=description):
# yield immediately so errors could be acted upon outside, before
# we continue
if not (res['type'] == 'dataset' and res['path'] in yielded_ds):
# unless we reported on this dataset before
if res['type'] == 'dataset':
# make a record, recursive below might now want to report
# a 'notneeded'
yielded_ds.append(res['path'])
yield res
# update to the current innermost dataset
containing_ds.append(res['path'])
if len(containing_ds) < 2:
# no subdataset was installed, hence if the path was unavailable
# before it still is, no need to bother git annex
ap.update(status='impossible',
message='path does not exist')
yield ap
continue
# important to only do the next for the innermost subdataset
# as the `recursive` logic below relies on that!
# set the correct parent, for a dataset this would be the second-last
# reported subdataset
ap.update(parentds=containing_ds[-1])
if containing_ds[-1] == ap['path']:
# the path actually refers to the last installed dataset
ap.update(parentds=containing_ds[-2],
process_content=get_data,
type='dataset')
to_get.append(ap)
# results of recursive installation of yet undiscovered datasets
rec_get = []
if recursive and not recursion_limit == 'existing':
# obtain any subdatasets underneath the paths given inside the
# subdatasets that we know already exist
# unless we do not want recursion into not-yet-installed datasets
for ap in sorted(to_get, key=lambda x: x['path']):
if ap['type'] not in ('dataset', 'directory') or not ap.get('raw_input', False):
# a non-directory cannot have content underneath
# also we do NOT want to recurse into anything that was specifically
# requested, to avoid duplication
continue
subds = Dataset(ap['path'] if ap['type'] == 'dataset' else ap['parentds'])
lgr.info(
"Installing %s%s recursively",
subds,
(" underneath %s" % ap['path']
if subds.path != ap['path']
else ""))
for res in _recursive_install_subds_underneath(
subds,
# `ap['path']` was explicitly given as input
# we count recursions from the input, hence we
# can start with the full number
recursion_limit,
reckless,
start=ap['path'],
refds_path=refds_path,
description=description):
# yield immediately so errors could be acted upon
# outside, before we continue
if not (res['type'] == 'dataset' and res['path'] in yielded_ds):
# unless we reported on this dataset before
if res['type'] == 'dataset':
# make a record
yielded_ds.append(res['path'])
yield res
if not (res['status'] == 'ok' and res['type'] == 'dataset'):
# not a dataset that was just installed, we just reported it
# upstairs, and can ignore it from now on
continue
# paranoia, so popular these days...
assert GitRepo.is_valid_repo(res['path'])
# keep a copy of the install record for `get` later on
get_ap = {k: v for k, v in res.items()
if not k == 'status'}
get_ap['process_content'] = get_data
rec_get.append(get_ap)
if not get_data:
# done already
return
# merge the two AP lists
to_get.extend(rec_get)
# sort into datasets
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
to_get,
refds_path=refds_path)
assert(not completed)
# hand over to git-annex, get files content,
# report files in git as 'notneeded' to get
for ds_path in sorted(content_by_ds.keys()):
ds = Dataset(ds_path)
# grab content, ignore subdataset entries
content = [ap['path'] for ap in content_by_ds[ds_path]
if ap.get('type', None) != 'dataset' or ap['path'] == ds.path]
if not content:
# cut this short should there be nothing
continue
# needs to be an annex to get content
if not isinstance(ds.repo, AnnexRepo):
for r in results_from_paths(
content, status='notneeded',
message="no dataset annex, content already present",
action='get', logger=lgr,
refds=refds_path):
yield r
continue
respath_by_status = {}
for res in ds.repo.get(
content,
options=['--from=%s' % source] if source else [],
jobs=jobs):
res = annexjson2result(res, ds, type='file', logger=lgr,
refds=refds_path)
success = success_status_map[res['status']]
# TODO: in case of some failed commands (e.g. get) there might
# be no path in the record. yoh has only vague idea of logic
# here so just checks for having 'path', but according to
# results_from_annex_noinfo, then it would be assumed that
# `content` was acquired successfully, which is not the case
if 'path' in res:
respath_by_status[success] = \
respath_by_status.get(success, []) + [res['path']]
yield res
for r in results_from_annex_noinfo(
ds,
content,
respath_by_status,
dir_fail_msg='could not get some content in %s %s',
noinfo_dir_msg='nothing to get from %s',
noinfo_file_msg='already present',
action='get',
logger=lgr,
refds=refds_path):
yield r