def __call__(
path=None,
message=None,
dataset=None,
version_tag=None,
recursive=False,
recursion_limit=None,
updated=False,
message_file=None,
to_git=None,
):
if message and message_file:
raise ValueError(
"Both a message and message file were specified for save()")
path = assure_list(path)
if message_file:
with open(message_file) as mfh:
message = mfh.read()
# we want 'normal' to achieve the most compact argument list
# for git calls
# untracked_mode = 'no' if updated else 'normal'
# TODO however, Repo.add() would refuse to add any dotfiles
# in a directory that is itself untracked, hence the only
# choice is to go with potentially crazy long lists
# until https://github.com/datalad/datalad/issues/1454
# has a resolution
untracked_mode = 'no' if updated else 'all'
# there are three basic scenarios:
# 1. save modifications to any already tracked content
# 2. save any content (including removal of deleted content)
# to bring things to a clean state
# 3. like (2), but only operate on a given subset of content
# identified by paths
# - all three have to work in conjunction with --recursive
# - the difference between (1) and (2) should be no more
# that a switch from --untracked=no to --untracked=all
# in Repo.save()
# we do not support
# - simultaneous operations on multiple datasets from disjoint
# dataset hierarchies, hence a single reference dataset must be
# identifiable from the either
# - curdir or
# - the `dataset` argument.
# This avoids complex annotation loops and hierarchy tracking.
# - any modification upwards from the root dataset
ds = require_dataset(dataset, check_installed=True, purpose='saving')
# use status() to do all discovery and annotation of paths
paths_by_ds = {}
for s in Status()(
# ATTN: it is vital to pass the `dataset` argument as it,
# and not a dataset instance in order to maintain the path
# semantics between here and the status() call
dataset=dataset,
path=path,
untracked=untracked_mode,
recursive=recursive,
recursion_limit=recursion_limit,
result_renderer='disabled'):
# fish out status dict for this parent dataset
ds_status = paths_by_ds.get(s['parentds'], {})
# reassemble path status info as repo.status() would have made it
ds_status[ut.Path(s['path'])] = \
{k: v for k, v in iteritems(s)
if k not in (
'path', 'parentds', 'refds', 'status', 'action',
'logger')}
paths_by_ds[s['parentds']] = ds_status
lgr.debug('Determined %i datasets for saving from input arguments',
len(paths_by_ds))
# figure out what datasets to process, start with the ones containing
# the paths that were given as arguments
discovered_datasets = list(paths_by_ds.keys())
if dataset:
# if a reference dataset was given we want to save all the way up
# to it, so let's throw it into the mix
discovered_datasets.append(ds.path)
# sort the datasets into (potentially) disjoint hierarchies,
# or a single one, if a reference dataset was given
dataset_hierarchies = get_tree_roots(discovered_datasets)
for rootds, children in iteritems(dataset_hierarchies):
edges = {}
discover_dataset_trace_to_targets(rootds,
children, [],
edges,
includeds=children)
for superds, subdss in iteritems(edges):
superds_status = paths_by_ds.get(superds, {})
for subds in subdss:
# TODO actually start from an entry that may already
# exist in the status record
superds_status[ut.Path(subds)] = dict(
# shot from the hip, some status config
# to trigger this specific super/sub
# relation to be saved
state='untracked',
type='dataset')
paths_by_ds[superds] = superds_status
# TODO parallelize, whenever we have multiple subdataset of a single
# dataset they can all be processed simultaneously
# sort list of dataset to handle, starting with the ones deep down
for pdspath in sorted(paths_by_ds, reverse=True):
pds = Dataset(pdspath)
# pop status for this dataset, we are not coming back to it
pds_status = {
# for handing over to the low-level code, we recode any
# path relative to the real repo location, this avoid
# cumbersome symlink handling without context in the
# lower levels
pds.repo.pathobj / p.relative_to(pdspath): props
for p, props in iteritems(paths_by_ds.pop(pdspath))
}
start_commit = pds.repo.get_hexsha()
if not all(p['state'] == 'clean' for p in pds_status.values()):
for res in pds.repo.save_(
message=message,
# make sure to have the `path` arg be None, as we want
# to prevent and bypass any additional repo.status()
# calls
paths=None,
# prevent whining of GitRepo
git=True
if not hasattr(ds.repo, 'annexstatus') else to_git,
# we are supplying the full status already, do not
# detect anything else
untracked='no',
_status=pds_status):
# TODO remove stringification when datalad-core can handle
# path objects, or when PY3.6 is the lowest supported
# version
for k in ('path', 'refds'):
if k in res:
res[k] = str(
# recode path back to dataset path anchor
pds.pathobj /
res[k].relative_to(pds.repo.pathobj))
yield res
# report on the dataset itself
dsres = dict(
action='save',
type='dataset',
path=pds.path,
refds=ds.path,
status='ok'
if start_commit != pds.repo.get_hexsha() else 'notneeded',
logger=lgr,
)
if not version_tag:
yield dsres
continue
try:
pds.repo.tag(version_tag)
dsres.update(status='ok', version_tag=version_tag)
yield dsres
except CommandError as e:
if dsres['status'] == 'ok':
# first we yield the result for the actual save
yield dsres.copy()
# and now complain that tagging didn't work
dsres.update(status='error',
message=('cannot tag this version: %s',
e.stderr.strip()))
yield dsres
def __call__(
path=None,
dataset=None,
# support passing this through in a path by path basis
to_git=None,
save=True,
message=None,
message_file=None,
recursive=False,
recursion_limit=None,
ds2super=False,
git_opts=None,
annex_opts=None,
annex_add_opts=None,
jobs=None):
# parameter constraints:
if not path:
raise InsufficientArgumentsError(
"insufficient information for adding: requires at least a path")
refds_path = Interface.get_refds_path(dataset)
common_report = dict(action='add', logger=lgr, refds=refds_path)
if message and message_file:
raise ValueError("Both a message and message file were specified")
if message_file:
with open(message_file, "rb") as mfh:
message = assure_unicode(mfh.read())
to_add = []
subds_to_add = {}
ds_to_annotate_from_recursion = {}
got_nothing = True
for ap in AnnotatePaths.__call__(
path=path,
dataset=dataset,
# never recursion, need to handle manually below to be able to
# discover untracked content
recursive=False,
action='add',
# speed things up by using Git's modification detection, if there
# is a repo with at least one commit
modified='HEAD' \
if dataset and \
GitRepo.is_valid_repo(refds_path) and \
GitRepo(refds_path).get_hexsha() \
else None,
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
return_type='generator',
on_failure='ignore'):
got_nothing = False
if ap.get('status', None):
# this is done
yield ap
continue
if ap.get('parentds', None) is None and ap.get('type', None) != 'dataset':
yield get_status_dict(
status='impossible',
message='"there is no dataset to add this path to',
**dict(common_report, **ap))
continue
if ap.get('type', None) == 'directory' and \
ap.get('state', None) == 'untracked' and \
GitRepo.is_valid_repo(ap['path']):
# this is an untracked wannabe subdataset in disguise
ap['type'] = 'dataset'
if recursive and \
(ap.get('raw_input', False) or
ap.get('state', None) in ('added', 'modified', 'untracked')) and \
(ap.get('parentds', None) or ap.get('type', None) == 'dataset'):
# this was an actually requested input path, or a path that was found
# modified by path annotation, based on an input argument
# we need to recurse into all subdirs to find potentially
# unregistered subdatasets
# but only if this path has a parent, or is itself a dataset
# otherwise there is nothing to add to
_discover_subdatasets_recursively(
ds_to_annotate_from_recursion,
ap['path'], [ap['parentds'] if 'parentds' in ap else ap['path']],
recursion_limit)
# get the file content of the root dataset of this search added too
# but be careful with extreme recursion_limit settings
if recursion_limit is None or recursion_limit > 0:
ap['process_content'] = True
# record for further processing
if not ap['path'] in ds_to_annotate_from_recursion:
# if it was somehow already discovered
to_add.append(ap)
if got_nothing:
# path annotation yielded nothing, most likely cause is that nothing
# was found modified, we need to say something about the reference
# dataset
yield get_status_dict(
'add',
status='notneeded',
path=refds_path,
type='dataset',
logger=lgr)
return
for subds in ds_to_annotate_from_recursion:
if subds not in subds_to_add:
# always prefer the already annotated path
subds_to_add[subds] = ds_to_annotate_from_recursion[subds]
if dataset:
# we have a base dataset, discover any intermediate datasets between
# the base and any already discovered dataset
discovered = {}
discover_dataset_trace_to_targets(
# from here
dataset.path,
# to any dataset we are aware of
subds_to_add.keys(),
[],
discovered)
for parentds in discovered:
for subds in discovered[parentds]:
subds_to_add[subds] = subds_to_add.get(
subds,
dict(path=subds, parentds=parentds, type='dataset'))
# merge custom paths and discovered dataset records, paths needs to go first,
# because we know most about then, and subsequent annotation call we skip the
# later duplicate ones
to_add.extend(subds_to_add.values())
# and compact, this should be OK as all the info is in each ap dict
to_add = unique(to_add, lambda x: x['path'])
if not to_add:
# nothing left to do, potentially all errored before
return
# now re-annotate all paths, this will be fast for already annotated ones
# and will amend the annotation for others, it will also deduplicate
annotated_paths = AnnotatePaths.__call__(
path=to_add,
dataset=dataset,
# never recursion, done already
recursive=False,
action='add',
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
return_type='generator',
# if there is an error now, we made this mistake in here
on_failure='stop')
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
annotated_paths,
refds_path=refds_path)
assert(not completed)
if not content_by_ds:
# we should have complained about any inappropriate path argument
# above, so if nothing is left, we can simply exit
return
# simple loop over datasets -- save happens later
# start deep down
to_save = []
for ds_path in sorted(content_by_ds, reverse=True):
ds = Dataset(ds_path)
torepoadd = {}
respath_by_status = {}
for ap in content_by_ds[ds_path]:
# we have a new story
ap.pop('status', None)
torepoadd[ap['path']] = ap
# skip anything that doesn't look like a wannabe subdataset
if not ap.get('type', None) == 'dataset' or \
ap['path'] == ds_path:
continue
if ap.get('registered_subds', False):
# subdataset that might be in this list because of the
# need to save all the way up to a super dataset
respath_by_status['success'] = \
respath_by_status.get('success', []) + [ap['path']]
yield get_status_dict(
status='notneeded',
message="already known subdataset",
**dict(common_report, **ap))
continue
subds = Dataset(ap['path'])
subds_relpath = relpath(ap['path'], ds_path)
# Register the repository in the repo tree as a submodule
try:
ds.repo.add_submodule(subds_relpath, url=None, name=None)
except (CommandError, InvalidGitRepositoryError) as e:
yield get_status_dict(
ds=subds, status='error', message=e.stderr,
**dict(common_report, **ap))
continue
# queue for saving using the updated annotated path
ap['registered_subds'] = True
# I hope this is true in direct mode too
# TODO this is disabled, because in some circumstances
# staging just doesn't happen, and it is unclear when
# exactly -- the case that prompted disabling was a submodule
# that had no content except for other submodules was not staged,
# whereas another submodule on the same level in the same
# superdataset which also has one file in it was staged
# disable to work correctly, while paying a little bit of
# slow down
#ap['staged'] = True
to_save.append(ap)
# report added subdatasets -- `annex add` below won't do it
yield get_status_dict(
ds=subds,
status='ok',
message='added new subdataset',
**dict(common_report, **ap))
# make sure that .gitmodules is added to the list of files
gitmodules_path = opj(ds.path, '.gitmodules')
# for git
torepoadd[gitmodules_path] = dict(path=gitmodules_path)
# and for save
to_save.append(dict(
path=gitmodules_path,
parentds=ds_path,
type='file'))
# make sure any last minute additions make it to the saving stage
# XXX? should content_by_ds become OrderedDict so that possible
# super here gets processed last?
lgr.debug('Adding content to repo %s: %s', ds.repo, torepoadd)
is_annex = isinstance(ds.repo, AnnexRepo)
add_kw = {'jobs': jobs} if is_annex and jobs else {}
added = ds.repo.add_(
list(torepoadd.keys()),
git=to_git if is_annex else True,
**add_kw
)
for a in added:
res = annexjson2result(a, ds, type='file', **common_report)
success = success_status_map[res['status']]
respath_by_status[success] = \
respath_by_status.get(success, []) + [res['path']]
# produce best possible path/result annotation
if res['path'] in torepoadd:
# pull out correct ap for any path that comes out here
# (that we know things about), and use the original annotation
# instead of just the annex report
res = dict(torepoadd[res['path']], **res)
# override this in all cases to be safe
res['parentds'] = ds.path
if success:
# this was successfully added, queue for saving this very path
# in the dataset
ap = {k: v for k, v in res.items() if k != 'status'}
ap['staged'] = True
# strip any status and state info (e.g. save will refuse to save
# stuff that is marked state='untracked'
to_save.append({k: v for k, v in res.items()
if k not in ('status', 'state')})
if a['file'] == '.gitmodules':
# filter out .gitmodules, because this is only included for
# technical reasons and has nothing to do with the actual content
continue
if GitRepo.is_valid_repo(res['path']):
# more accurate report in case of an added submodule
# mountpoint.
# XXX Actually not sure if this can really happen
# (depends on what our low-level code would do)
# but worst case is that we loose a little bit of
# coverage...
res['type'] = 'dataset'
res['message'] = 'added new state as submodule'
yield res
for r in results_from_annex_noinfo(
ds, torepoadd, respath_by_status,
dir_fail_msg='could not add some content in %s %s',
noinfo_dir_msg='nothing to add from %s',
noinfo_file_msg='already included in the dataset',
action='add',
logger=lgr,
refds=refds_path):
if r['path'] in torepoadd:
# pull out correct ap for any path that comes out here
# (that we know things about), and use the original annotation
# instead of just the annex report
r = dict(r, **torepoadd[r['path']])
if r['status'] == 'notneeded':
# this could be a file that was staged already, it doesn't need
# to be added, but it should be saved/commited if so desired
to_save.append({k: v for k, v in r.items()
if k not in ('status', 'state')})
# XXX something is fishy with the next one, rethink when sober....
if r['path'] == ds_path and r['status'] == 'ok':
# this is for the entire dataset itself which was explicitly requested
# make sure to save all
r['type'] = 'dataset'
r['process_content'] = True
to_save.append({k: v for k, v in r.items() if k != 'status'})
yield r
if refds_path and ds_path != refds_path and len(respath_by_status.get('success', [])):
# TODO XXX we have an issue here when with `add('.')` and annex ignores any
# dotfiles. In this case we end up not saving a dataset completely, because
# we rely on accurate reporting. there is an issue about this already
# TODO look up the issue ID
# if there is a base dataset, but we are below it, and we have anything done to this
# dataset -> queue dataset itself for saving its state in the parent
ds_ap = dict(
path=ds.path,
# we have to look for the parent here, as we must save the
# subdataset in the parent and not the whole subdataset itself
type='dataset')
parentds = get_dataset_root(normpath(opj(ds.path, pardir)))
if parentds:
ds_ap['parentds'] = parentds
if dataset:
ds_ap['refds'] = refds_path
to_save.append(ds_ap)
if not save:
lgr.debug('Not calling `save` as instructed')
return
# TODO tell save what was staged already! Set 'staged=True' for
# respective annotated paths that are fed into `save`
# do not reuse any of the sorting done in here for saving, but instead
# pass on all the annotated paths to have `save` figure out what to do with
# them -- this is costs something, but should be safer, and frankly is
# more comprehensible
for res in Save.__call__(
# hand-selected annotated paths
path=to_save,
dataset=refds_path,
message=message if message else '[DATALAD] added content',
return_type='generator',
result_xfm=None,
result_filter=None,
on_failure='ignore'):
yield res
def __call__(
path=None,
*,
dataset=None,
recursive=False,
recursion_limit=None,
update_mode='target',
incremental=False,
force_extraction=False,
save=True):
refds_path = require_dataset(dataset)
# it really doesn't work without a dataset
ds = require_dataset(
dataset, check_installed=True, purpose='metadata aggregation')
path = ensure_list(path)
if not path:
# then current/reference dataset is "aggregated"
# We should not add ds.path always since then --recursive would
# also recurse current even if paths are given
path.append(ds.path)
agginfo_db_location, agg_base_path = get_ds_aggregate_db_locations(
ds,
# do not warn here, next call triggers the same warning
warn_absent=False)
agginfo_db = load_ds_aggregate_db(ds, abspath=True)
to_save = []
to_aggregate = set()
paths_by_ds, errors = get_paths_by_ds(
require_dataset(dataset),
dataset,
paths=ensure_list(path),
subdsroot_mode='super')
for ap in _minimal_annotate_paths(
paths_by_ds,
errors,
action='aggregate_metadata',
recursive=recursive,
recursion_limit=recursion_limit):
if ap.get('status', None):
# this is done
yield ap
continue
ap_type = ap.get('type', None)
ap_state = ap.get('state', None)
assert('parentds' in ap or ap_type == 'dataset')
if ap_type == 'dataset' and ap_state != 'absent':
# a present dataset, we can take directly from it
aggsrc = ap['path']
lgr.info('Aggregate metadata for dataset %s', aggsrc)
else:
# everything else needs to come from the parent
aggsrc = ap['parentds']
if ap_state == 'absent':
lgr.info(
'Attempt to use pre-aggregate metadata for absent %s from dataset at %s',
ap['path'],
aggsrc)
else:
lgr.info(
'Aggregate metadata for %s from dataset at %s',
ap['path'],
aggsrc)
to_aggregate.add(aggsrc)
if ap_state == 'absent':
# key thought: recursive is done by path annotation, hence
# once we hit an absent dataset, we are 100% certain that
# there is nothing to recurse into on the file system
# hence we only have to look into the aggregated metadata
# of the last available dataset in the dataset tree edge
#
# if there is nothing at this path, we need to look into the
# parentds and check if we know anything about this path
# if we do, we need to grab all the info and objects
# if not, we need to error
res = _get_dsinfo_from_aggmetadata(
aggsrc, ap['path'], recursive, agginfo_db)
if not isinstance(res, list):
yield get_status_dict(
status='impossible',
message=res,
action='aggregate_metadata',
path=ap['path'],
logger=lgr)
continue
# cue for aggregation
to_aggregate.update(res)
else:
# actually aggregate metadata for this dataset, immediately place
# generated objects into the aggregated or reference dataset,
# and put info into DB to get the distributed to all datasets
# that need to be updated
errored = _dump_extracted_metadata(
ds,
Dataset(aggsrc),
agginfo_db,
to_save,
force_extraction,
agg_base_path)
if errored:
yield get_status_dict(
status='error',
message='Metadata extraction failed (see previous error message, set datalad.runtime.raiseonerror=yes to fail immediately)',
action='aggregate_metadata',
path=aggsrc,
logger=lgr)
# at this point we have dumped all aggregated metadata into object files
# somewhere, we know what needs saving, but having saved anything, and
# we know about the states of all aggregated dataset in the DB
# what remains to do is to update all dataset, so they have there own copy
# of aggregated metadata and update their respective aggregate.json with
# info on what states we just aggregated from
# first, let's figure out what dataset need updating at all
# get adjencency info of the dataset tree spanning the base to all leaf dataset
# associated with the path arguments
if update_mode == 'all':
ds_adj = {}
discover_dataset_trace_to_targets(
ds.path, to_aggregate, [], ds_adj,
# we know that to_aggregate only lists datasets, existing and
# absent ones -- we want to aggregate all of them, either from
# just extracted metadata, or from previously aggregated metadata
# of the closest superdataset
includeds=to_aggregate)
# TODO we need to work in the info about dataset that we only got from
# aggregated metadata, that had no trace on the file system in here!!
subtrees = _adj2subtrees(ds.path, ds_adj, to_aggregate)
elif update_mode == 'target':
subtrees = {ds.path: list(agginfo_db.keys())}
else:
raise ValueError(
"unknown `update_mode` '%s' for metadata aggregation", update_mode)
# go over datasets in bottom-up fashion
for parentds_path in sorted(subtrees, reverse=True):
lgr.info('Update aggregate metadata in dataset at: %s', parentds_path)
_update_ds_agginfo(
ds.path,
parentds_path,
subtrees[parentds_path],
incremental,
agginfo_db,
to_save)
# update complete
res = get_status_dict(
status='ok',
action='aggregate_metadata',
path=parentds_path,
type='dataset',
logger=lgr)
res.update(agginfo_db.get(parentds_path, {}))
yield res
#
# save potential modifications to dataset global metadata
#
if not to_save:
return
lgr.info('Attempting to save %i files/datasets', len(to_save))
for res in Save.__call__(
# save does not need any pre-annotated path hints
path=[r['path'] for r in to_save],
dataset=refds_path,
message='[DATALAD] Dataset aggregate metadata update',
return_type='generator',
result_renderer='disabled',
result_xfm=None,
result_filter=None,
on_failure='ignore'):
yield res
def __call__(
path=None,
dataset=None,
# support passing this through in a path by path basis
to_git=None,
save=True,
message=None,
recursive=False,
recursion_limit=None,
ds2super=False,
git_opts=None,
annex_opts=None,
annex_add_opts=None,
jobs=None):
# parameter constraints:
if not path:
raise InsufficientArgumentsError(
"insufficient information for adding: requires at least a path"
)
refds_path = Interface.get_refds_path(dataset)
common_report = dict(action='add', logger=lgr, refds=refds_path)
to_add = []
subds_to_add = {}
ds_to_annotate_from_recursion = {}
got_nothing = True
for ap in AnnotatePaths.__call__(
path=path,
dataset=dataset,
# never recursion, need to handle manually below to be able to
# discover untracked content
recursive=False,
action='add',
# speed things up by using Git's modification detection, if there
# is a repo with at least one commit
modified='HEAD' \
if dataset and \
GitRepo.is_valid_repo(refds_path) and \
GitRepo(refds_path).get_hexsha() \
else None,
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
return_type='generator',
on_failure='ignore'):
got_nothing = False
if ap.get('status', None):
# this is done
yield ap
continue
if ap.get('parentds', None) is None and ap.get('type',
None) != 'dataset':
yield get_status_dict(
status='impossible',
message='"there is no dataset to add this path to',
**dict(common_report, **ap))
continue
if ap.get('type', None) == 'directory' and \
ap.get('state', None) == 'untracked' and \
GitRepo.is_valid_repo(ap['path']):
# this is an untracked wannabe subdataset in disguise
ap['type'] = 'dataset'
if recursive and \
(ap.get('raw_input', False) or
ap.get('state', None) in ('added', 'modified', 'untracked')) and \
(ap.get('parentds', None) or ap.get('type', None) == 'dataset'):
# this was an actually requested input path, or a path that was found
# modified by path annotation, based on an input argument
# we need to recurse into all subdirs to find potentially
# unregistered subdatasets
# but only if this path has a parent, or is itself a dataset
# otherwise there is nothing to add to
_discover_subdatasets_recursively(
ds_to_annotate_from_recursion, ap['path'],
[ap['parentds'] if 'parentds' in ap else ap['path']],
recursion_limit)
# get the file content of the root dataset of this search added too
# but be careful with extreme recursion_limit settings
if recursion_limit is None or recursion_limit > 0:
ap['process_content'] = True
# record for further processing
if not ap['path'] in ds_to_annotate_from_recursion:
# if it was somehow already discovered
to_add.append(ap)
# TODO check if next isn't covered by discover_dataset_trace_to_targets already??
if dataset and ap.get('type', None) == 'dataset':
# duplicates not possible, annotated_paths returns unique paths
subds_to_add[ap['path']] = ap
if got_nothing:
# path annotation yielded nothing, most likely cause is that nothing
# was found modified, we need to say something about the reference
# dataset
yield get_status_dict('add',
status='notneeded',
path=refds_path,
type='dataset',
logger=lgr)
return
for subds in ds_to_annotate_from_recursion:
if subds not in subds_to_add:
# always prefer the already annotated path
subds_to_add[subds] = ds_to_annotate_from_recursion[subds]
if dataset:
# we have a base dataset, discover any intermediate datasets between
# the base and any already discovered dataset
discovered = {}
discover_dataset_trace_to_targets(
# from here
dataset.path,
# to any dataset we are aware of
subds_to_add.keys(),
[],
discovered)
for parentds in discovered:
for subds in discovered[parentds]:
subds_to_add[subds] = subds_to_add.get(
subds,
dict(path=subds, parentds=parentds, type='dataset'))
# merge custom paths and discovered dataset records, paths needs to go first,
# because we know most about then, and subsequent annotation call we skip the
# later duplicate ones
to_add.extend(subds_to_add.values())
# and compact, this should be OK as all the info is in each ap dict
to_add = unique(to_add, lambda x: x['path'])
if not to_add:
# nothing left to do, potentially all errored before
return
# now re-annotate all paths, this will be fast for already annotated ones
# and will amend the annotation for others, it will also deduplicate
annotated_paths = AnnotatePaths.__call__(
path=to_add,
dataset=dataset,
# never recursion, done already
recursive=False,
action='add',
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
return_type='generator',
# if there is an error now, we made this mistake in here
on_failure='stop')
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
annotated_paths,
refds_path=refds_path)
assert (not completed)
if not content_by_ds:
# we should have complained about any inappropriate path argument
# above, so if nothing is left, we can simply exit
return
# simple loop over datasets -- save happens later
# start deep down
to_save = []
for ds_path in sorted(content_by_ds, reverse=True):
ds = Dataset(ds_path)
torepoadd = {}
respath_by_status = {}
for ap in content_by_ds[ds_path]:
# we have a new story
ap.pop('status', None)
torepoadd[ap['path']] = ap
# skip anything that doesn't look like a wannabe subdataset
if not ap.get('type', None) == 'dataset' or \
ap['path'] == ds_path:
continue
if ap.get('registered_subds', False):
# subdataset that might be in this list because of the
# need to save all the way up to a super dataset
respath_by_status['success'] = \
respath_by_status.get('success', []) + [ap['path']]
yield get_status_dict(status='notneeded',
message="already known subdataset",
**dict(common_report, **ap))
continue
subds = Dataset(ap['path'])
# check that the subds has a commit, and refuse
# to operate on it otherwise, or we would get a bastard
# submodule that cripples git operations
if not subds.repo.get_hexsha():
yield get_status_dict(
ds=subds,
status='impossible',
message='cannot add subdataset with no commits',
**dict(common_report, **ap))
continue
subds_relpath = relpath(ap['path'], ds_path)
# make an attempt to configure a submodule source URL based on the
# discovered remote configuration
remote, branch = subds.repo.get_tracking_branch()
subds_url = subds.repo.get_remote_url(
remote) if remote else None
# Register the repository in the repo tree as a submodule
try:
ds.repo.add_submodule(subds_relpath,
url=subds_url,
name=None)
except CommandError as e:
yield get_status_dict(ds=subds,
status='error',
message=e.stderr,
**dict(common_report, **ap))
continue
# queue for saving using the updated annotated path
ap['registered_subds'] = True
# I hope this is true in direct mode too
# TODO this is disabled, because in some circumstances
# staging just doesn't happen, and it is unclear when
# exactly -- the case that prompted disabling was a submodule
# that had no content except for other submodules was not staged,
# whereas another submodule on the same level in the same
# superdataset which also has one file in it was staged
# disable to work correctly, while paying a little bit of
# slow down
#ap['staged'] = True
to_save.append(ap)
_fixup_submodule_dotgit_setup(ds, subds_relpath)
# report added subdatasets -- `annex add` below won't do it
yield get_status_dict(ds=subds,
status='ok',
message='added new subdataset',
**dict(common_report, **ap))
# make sure that .gitmodules is added to the list of files
gitmodules_path = opj(ds.path, '.gitmodules')
# for git
torepoadd[gitmodules_path] = dict(path=gitmodules_path)
# and for save
to_save.append(
dict(path=gitmodules_path, parentds=ds_path, type='file'))
# make sure any last minute additions make it to the saving stage
# XXX? should content_by_ds become OrderedDict so that possible
# super here gets processed last?
lgr.debug('Adding content to repo %s: %s', ds.repo, torepoadd)
is_annex = isinstance(ds.repo, AnnexRepo)
add_kw = {'jobs': jobs} if is_annex and jobs else {}
added = ds.repo.add(list(torepoadd.keys()),
git=to_git if is_annex else True,
**add_kw)
for a in added:
res = annexjson2result(a, ds, type='file', **common_report)
success = success_status_map[res['status']]
respath_by_status[success] = \
respath_by_status.get(success, []) + [res['path']]
# produce best possible path/result annotation
if res['path'] in torepoadd:
# pull out correct ap for any path that comes out here
# (that we know things about), and use the original annotation
# instead of just the annex report
res = dict(torepoadd[res['path']], **res)
# override this in all cases to be safe
res['parentds'] = ds.path
if success:
# this was successfully added, queue for saving this very path
# in the dataset
ap = {k: v for k, v in res.items() if k != 'status'}
ap['staged'] = True
# strip any status and state info (e.g. save will refuse to save
# stuff that is marked state='untracked'
to_save.append({
k: v
for k, v in res.items() if k not in ('status', 'state')
})
if a['file'] == '.gitmodules':
# filter out .gitmodules, because this is only included for
# technical reasons and has nothing to do with the actual content
continue
if GitRepo.is_valid_repo(res['path']):
# more accurate report in case of an added submodule
# mountpoint.
# XXX Actually not sure if this can really happen
# (depends on what our low-level code would do)
# but worst case is that we loose a little bit of
# coverage...
res['type'] = 'dataset'
res['message'] = 'added new state as submodule'
yield res
for r in results_from_annex_noinfo(
ds,
torepoadd,
respath_by_status,
dir_fail_msg='could not add some content in %s %s',
noinfo_dir_msg='nothing to add from %s',
noinfo_file_msg='already included in the dataset',
action='add',
logger=lgr,
refds=refds_path):
if r['path'] in torepoadd:
# pull out correct ap for any path that comes out here
# (that we know things about), and use the original annotation
# instead of just the annex report
r = dict(r, **torepoadd[r['path']])
if r['status'] == 'notneeded':
# this could be a file that was staged already, it doesn't need
# to be added, but it should be saved/commited if so desired
to_save.append({
k: v
for k, v in r.items() if k not in ('status', 'state')
})
# XXX something is fishy with the next one, rethink when sober....
if r['path'] == ds_path and r['status'] == 'ok':
# this is for the entire dataset itself which was explicitly requested
# make sure to save all
r['type'] = 'dataset'
r['process_content'] = True
to_save.append(
{k: v
for k, v in r.items() if k != 'status'})
yield r
if refds_path and ds_path != refds_path and len(
respath_by_status.get('success', [])):
# TODO XXX we have an issue here when with `add('.')` and annex ignores any
# dotfiles. In this case we end up not saving a dataset completely, because
# we rely on accurate reporting. there is an issue about this already
# TODO look up the issue ID
# if there is a base dataset, but we are below it, and we have anything done to this
# dataset -> queue dataset itself for saving its state in the parent
ds_ap = dict(
path=ds.path,
# we have to look for the parent here, as we must save the
# subdataset in the parent and not the whole subdataset itself
type='dataset')
parentds = get_dataset_root(normpath(opj(ds.path, pardir)))
if parentds:
ds_ap['parentds'] = parentds
if dataset:
ds_ap['refds'] = refds_path
to_save.append(ds_ap)
if not save:
lgr.debug('Not calling `save` as instructed')
return
# TODO tell save what was staged already! Set 'staged=True' for
# respective annotated paths that are fed into `save`
# do not reuse any of the sorting done in here for saving, but instead
# pass on all the annotated paths to have `save` figure out what to do with
# them -- this is costs something, but should be safer, and frankly is
# more comprehensible
for res in Save.__call__(
# hand-selected annotated paths
path=to_save,
dataset=refds_path,
message=message if message else '[DATALAD] added content',
return_type='generator',
result_xfm=None,
result_filter=None,
on_failure='ignore'):
yield res
def __call__(message=None,
files=None,
dataset=None,
all_updated=True,
all_changes=None,
version_tag=None,
recursive=False,
recursion_limit=None,
super_datasets=False):
if all_changes is not None:
from datalad.support.exceptions import DeprecatedError
raise DeprecatedError(
new="all_updated option where fits and/or datalad add",
version="0.5.0",
msg="RF: all_changes option passed to the save")
if not dataset and not files:
# we got nothing at all -> save what is staged in the repo in "this" directory?
# we verify that there is an actual repo next
dataset = abspath(curdir)
refds_path = Interface.get_refds_path(dataset)
to_process = []
for ap in AnnotatePaths.__call__(
path=files,
dataset=refds_path,
recursive=recursive,
recursion_limit=recursion_limit,
action='save',
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
return_type='generator',
on_failure='ignore'):
# next check should not be done during annotation, as it is possibly expensive
# and not generally useful
if ap.get('status', None) == 'impossible' and \
ap.get('state', None) == 'absent' and \
ap.get('parentds', None):
# this is not here anymore, but it might actually have been a deleted
# component
if relpath(ap['path'], start=ap['parentds']) \
in Dataset(ap['parentds']).repo.get_deleted_files():
# ok, this is a staged deletion that we want to save
ap['status'] = ''
del ap['message']
if ap.get('status', None):
# this is done
yield ap
continue
# for things like: `ds.save()`
# or recursively discovered datasets
if ap['path'] == refds_path or \
(ap.get('type', None) == 'dataset' and
not ap.get('raw_input', False) and
not ap.get('state', None) == 'absent'):
ap['process_content'] = True
ap['process_updated_only'] = all_updated
to_process.append(ap)
if not to_process:
# nothing left to do, potentially all errored before
return
if super_datasets:
# search for the topmost superdatasets of any path
dss = [
Dataset(ap.get('parentds', ap['path'])) for ap in to_process
]
superdss = [ds.get_superdataset(topmost=True) for ds in dss]
superdss = get_tree_roots(
unique(ds.path for ds in dss + superdss if ds))
if dataset:
# need to adjust the reference to the new superds
# if we had one ref before, we should still have exactly one
assert len(superdss) <= 1
dataset = list(superdss.keys())[0]
refds_path = dataset
elif refds_path:
# there is a single superdataset
superdss = {
refds_path:
unique(
[ap['parentds'] for ap in to_process if 'parentds' in ap])
}
else:
# sort all datasets under their potential superdatasets
# start from the top to get all subdatasets down the line
# and collate them into as few superdatasets as possible
# this is quick, just string operations
superdss = get_tree_roots(
unique(
[ap['parentds'] for ap in to_process if 'parentds' in ap]))
# for each "superdataset" check the tree of subdatasets and make sure
# we gather all datasets between the super and any subdataset
# so we can save them all bottom-up in order to be able to properly
# save the superdataset
# if this is called from e.g. `add` this is actually not necessary,
# but in the general case we cannot avoid it
# TODO maybe introduce a switch?
discovered = {}
for superds_path in superdss:
target_subs = superdss[superds_path]
discover_dataset_trace_to_targets(
# from here
superds_path,
# to all
target_subs,
[],
discovered)
# create a new minimally annotated path for each discovered dataset
discovered_added = set()
for parentds in discovered:
for subds in discovered[parentds]:
to_process.append(
dict(path=subds, parentds=parentds, type='dataset'))
discovered_added.add(subds)
# make sure we have an entry for each dataset, including those
# tha are just parents
for parentds in discovered:
if parentds not in discovered_added:
to_process.append(
dict(
path=parentds,
type='dataset',
# make sure we save content of superds later on
process_content=True))
# now re-annotate all paths, this will be fast for already annotated ones
# and will amend the annotation for others, deduplication happens here too
annotated_paths = AnnotatePaths.__call__(
path=to_process,
dataset=dataset,
# never recursion, done already
recursive=False,
action='save',
unavailable_path_status='',
nondataset_path_status='impossible',
return_type='generator',
# if there is an error now, we made this mistake in here
on_failure='stop')
# now sort into datasets so we can process them one by one
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
annotated_paths,
refds_path=refds_path,
path_only=False)
assert (not completed)
# iterate over all datasets, starting at the bottom
for dspath in sorted(content_by_ds.keys(), reverse=True):
ds = Dataset(dspath)
res = get_status_dict('save', ds=ds, logger=lgr)
if not ds.is_installed():
# TODO This is likely impossible now
res['status'] = 'impossible'
res['message'] = ('dataset %s is not installed', ds)
yield res
continue
saved_state = save_dataset(ds,
content_by_ds[dspath],
message=message,
version_tag=version_tag)
if saved_state:
res['status'] = 'ok'
else:
res['status'] = 'notneeded'
yield res
def __call__(path=None,
dataset=None,
merge_native='init',
recursive=False,
recursion_limit=None,
save=True):
refds_path = Interface.get_refds_path(dataset)
# it really doesn't work without a dataset
ds = require_dataset(dataset,
check_installed=True,
purpose='metadata aggregation')
# always include the reference dataset
path = assure_list(path)
path.append(ds.path)
agginfo_db = {}
to_save = []
to_aggregate = set()
for ap in AnnotatePaths.__call__(
dataset=refds_path,
path=path,
recursive=recursive,
recursion_limit=recursion_limit,
action='aggregate_metadata',
# uninstalled subdatasets could be queried via aggregated metadata
# -> no 'error'
unavailable_path_status='',
nondataset_path_status='error',
return_type='generator',
on_failure='ignore'):
if ap.get('status', None):
# this is done
yield ap
continue
ap_type = ap.get('type', None)
ap_state = ap.get('state', None)
assert ('parentds' in ap or ap_type == 'dataset')
if ap_type == 'dataset' and ap_state != 'absent':
# a present dataset, we can take directly from it
aggsrc = ap['path']
lgr.info('Aggregate metadata for dataset %s', aggsrc)
else:
# everything else needs to come from the parent
aggsrc = ap['parentds']
if ap_state == 'absent':
lgr.info(
'Attempt to use pre-aggregate metadata for absent %s from dataset at %s',
ap['path'], aggsrc)
else:
lgr.info('Aggregate metadata for %s from dataset at %s',
ap['path'], aggsrc)
to_aggregate.add(aggsrc)
if ap_state == 'absent':
# key thought: recursive is done by path annotation, hence
# once we hit an absent dataset, we are 100% certain that
# there is nothing to recurse into on the file system
# hence we only have to look into the aggregated metadata
# of the last available dataset in the dataset tree edge
#
# if there is nothing at this path, we need to look into the
# parentds and check if we know anything about this path
# if we do, we need to grab all the info and objects
# if not, we need to error
res = _get_dsinfo_from_aggmetadata(aggsrc, ap['path'],
recursive, agginfo_db)
if not isinstance(res, list):
yield get_status_dict(status='impossible',
message=res,
action='aggregate_metadata',
path=ap['path'],
logger=lgr)
continue
# cue for aggregation
to_aggregate.update(res)
else:
# actually aggregate metadata for this dataset, immediately place
# generated objects into the aggregated or reference dataset,
# and put info into DB to get the distributed to all datasets
# that need to be updated
errored = _extract_metadata(ds, Dataset(aggsrc), agginfo_db,
merge_native, to_save)
if errored:
yield get_status_dict(
status='error',
message=
'Metadata extraction failed (see previous error message)',
action='aggregate_metadata',
path=aggsrc,
logger=lgr)
# at this point we have dumped all aggregated metadata into object files
# somewhere, we know what needs saving, but having saved anything, and
# we know about the states of all aggregated dataset in the DB
# what remains to do is to update all dataset, so they have there own copy
# of aggregated metadata and update their respective aggregate.json with
# info on what states we just aggregated from
# first, let's figure out what dataset need updating at all
# get adjencency info of the dataset tree spanning the base to all leaf dataset
# associated with the path arguments
ds_adj = {}
discover_dataset_trace_to_targets(ds.path, to_aggregate, [], ds_adj)
# TODO we need to work in the info about dataset that we only got from
# aggregated metadata, that had no trace on the file system in here!!
subtrees = _adj2subtrees(ds.path, ds_adj, to_aggregate)
# go over datasets in bottom-up fashion
for parentds_path in sorted(subtrees, reverse=True):
lgr.info('Update aggregate metadata in dataset at: %s',
parentds_path)
_update_ds_agginfo(ds.path, parentds_path, subtrees[parentds_path],
agginfo_db, to_save)
# update complete
yield get_status_dict(status='ok',
action='aggregate_metadata',
path=parentds_path,
type='dataset',
logger=lgr)
#
# save potential modifications to dataset global metadata
#
if not to_save:
return
lgr.info('Attempting to save %i files/datasets', len(to_save))
for res in Save.__call__(
path=to_save,
dataset=refds_path,
message='[DATALAD] dataset aggregate metadata update',
return_type='generator',
result_xfm=None,
result_filter=None,
on_failure='ignore'):
yield res
def __call__(path=None, message=None, dataset=None,
version_tag=None,
recursive=False, recursion_limit=None,
updated=False,
message_file=None,
to_git=None,
):
if message and message_file:
raise ValueError(
"Both a message and message file were specified for save()")
path = assure_list(path)
if message_file:
with open(message_file) as mfh:
message = mfh.read()
# we want 'normal' to achieve the most compact argument list
# for git calls
# untracked_mode = 'no' if updated else 'normal'
# TODO however, Repo.add() would refuse to add any dotfiles
# in a directory that is itself untracked, hence the only
# choice is to go with potentially crazy long lists
# until https://github.com/datalad/datalad/issues/1454
# has a resolution
untracked_mode = 'no' if updated else 'all'
# there are three basic scenarios:
# 1. save modifications to any already tracked content
# 2. save any content (including removal of deleted content)
# to bring things to a clean state
# 3. like (2), but only operate on a given subset of content
# identified by paths
# - all three have to work in conjunction with --recursive
# - the difference between (1) and (2) should be no more
# that a switch from --untracked=no to --untracked=all
# in Repo.save()
# we do not support
# - simultaneous operations on multiple datasets from disjoint
# dataset hierarchies, hence a single reference dataset must be
# identifiable from the either
# - curdir or
# - the `dataset` argument.
# This avoids complex annotation loops and hierarchy tracking.
# - any modification upwards from the root dataset
ds = require_dataset(dataset, check_installed=True, purpose='saving')
# use status() to do all discovery and annotation of paths
paths_by_ds = {}
for s in Status()(
# ATTN: it is vital to pass the `dataset` argument as it,
# and not a dataset instance in order to maintain the path
# semantics between here and the status() call
dataset=dataset,
path=path,
untracked=untracked_mode,
recursive=recursive,
recursion_limit=recursion_limit,
result_renderer='disabled'):
# fish out status dict for this parent dataset
ds_status = paths_by_ds.get(s['parentds'], {})
# reassemble path status info as repo.status() would have made it
ds_status[ut.Path(s['path'])] = \
{k: v for k, v in iteritems(s)
if k not in (
'path', 'parentds', 'refds', 'status', 'action',
'logger')}
paths_by_ds[s['parentds']] = ds_status
lgr.debug('Determined %i datasets for saving from input arguments',
len(paths_by_ds))
# figure out what datasets to process, start with the ones containing
# the paths that were given as arguments
discovered_datasets = list(paths_by_ds.keys())
if dataset:
# if a reference dataset was given we want to save all the way up
# to it, so let's throw it into the mix
discovered_datasets.append(ds.path)
# sort the datasets into (potentially) disjoint hierarchies,
# or a single one, if a reference dataset was given
dataset_hierarchies = get_tree_roots(discovered_datasets)
for rootds, children in iteritems(dataset_hierarchies):
edges = {}
discover_dataset_trace_to_targets(
rootds, children, [], edges, includeds=children)
for superds, subdss in iteritems(edges):
superds_status = paths_by_ds.get(superds, {})
for subds in subdss:
# TODO actually start from an entry that may already
# exist in the status record
superds_status[ut.Path(subds)] = dict(
# shot from the hip, some status config
# to trigger this specific super/sub
# relation to be saved
state='untracked',
type='dataset')
paths_by_ds[superds] = superds_status
# TODO parallelize, whenever we have multiple subdataset of a single
# dataset they can all be processed simultaneously
# sort list of dataset to handle, starting with the ones deep down
for pdspath in sorted(paths_by_ds, reverse=True):
pds = Dataset(pdspath)
# pop status for this dataset, we are not coming back to it
pds_status = {
# for handing over to the low-level code, we recode any
# path relative to the real repo location, this avoid
# cumbersome symlink handling without context in the
# lower levels
pds.repo.pathobj / p.relative_to(pdspath): props
for p, props in iteritems(paths_by_ds.pop(pdspath))}
start_commit = pds.repo.get_hexsha()
if not all(p['state'] == 'clean' for p in pds_status.values()):
for res in pds.repo.save_(
message=message,
# make sure to have the `path` arg be None, as we want
# to prevent and bypass any additional repo.status()
# calls
paths=None,
# prevent whining of GitRepo
git=True if not hasattr(ds.repo, 'annexstatus')
else to_git,
# we are supplying the full status already, do not
# detect anything else
untracked='no',
_status=pds_status):
# TODO remove stringification when datalad-core can handle
# path objects, or when PY3.6 is the lowest supported
# version
for k in ('path', 'refds'):
if k in res:
res[k] = text_type(
# recode path back to dataset path anchor
pds.pathobj / res[k].relative_to(
pds.repo.pathobj)
)
yield res
# report on the dataset itself
dsres = dict(
action='save',
type='dataset',
path=pds.path,
refds=ds.path,
status='ok'
if start_commit != pds.repo.get_hexsha()
else 'notneeded',
logger=lgr,
)
if not version_tag:
yield dsres
continue
try:
pds.repo.tag(version_tag)
dsres.update(
status='ok',
version_tag=version_tag)
yield dsres
except CommandError as e:
if dsres['status'] == 'ok':
# first we yield the result for the actual save
yield dsres.copy()
# and now complain that tagging didn't work
dsres.update(
status='error',
message=('cannot tag this version: %s', e.stderr.strip()))
yield dsres
def __call__(message=None, path=None, dataset=None,
all_updated=True, version_tag=None,
recursive=False, recursion_limit=None, super_datasets=False,
message_file=None
):
if not dataset and not path:
# we got nothing at all -> save what is staged in the repo in "this" directory?
# make sure we don't treat this as a user-provided '.' argument
path = [{'path': abspath(curdir), 'raw_input': False}]
refds_path = Interface.get_refds_path(dataset)
if message and message_file:
raise ValueError("Both a message and message file were specified")
if message_file:
with open(message_file, "rb") as mfh:
message = assure_unicode(mfh.read())
to_process = []
got_nothing = True
for ap in AnnotatePaths.__call__(
path=path,
dataset=refds_path,
recursive=recursive,
recursion_limit=recursion_limit,
action='save',
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
modified='HEAD' if not path and recursive else None,
return_type='generator',
on_failure='ignore'):
if ap.get('state', None) == 'untracked' and not ap.get('raw_input', False):
# this path was found untracked, but not explicitly given to save
# we will silently ignore this
continue
got_nothing = False
# next check should not be done during annotation, as it is possibly expensive
# and not generally useful
if ap.get('status', None) == 'impossible' and \
ap.get('state', None) == 'absent' and \
ap.get('parentds', None):
# this is not here anymore, but it might actually have been a deleted
# component
if relpath(ap['path'], start=ap['parentds']) \
in Dataset(ap['parentds']).repo.get_deleted_files():
# ok, this is a staged deletion that we want to save
ap['status'] = ''
del ap['message']
if ap.get('status', None):
# this is done
yield ap
continue
# for things like: `ds.save()`
# or recursively discovered datasets
if ap['path'] == refds_path or \
(ap.get('type', None) == 'dataset' and
not ap.get('raw_input', False) and
not ap.get('state', None) == 'absent'):
ap['process_content'] = True
ap['process_updated_only'] = all_updated
to_process.append(ap)
lgr.log(2, "save, to_process=%r", to_process)
if got_nothing and recursive and refds_path:
# path annotation yielded nothing, most likely cause is that nothing
# was found modified, we need to say something about the reference
# dataset
yield get_status_dict(
'save',
status='notneeded',
path=refds_path,
type='dataset',
logger=lgr)
return
if not to_process:
# nothing left to do, potentially all errored before
return
if super_datasets:
# search for the topmost superdatasets of any path
dss = [Dataset(ap.get('parentds', ap['path'])) for ap in to_process]
superdss = [ds.get_superdataset(topmost=True)
for ds in dss]
superdss = get_tree_roots(
unique(ds.path for ds in dss + superdss if ds))
if dataset:
# need to adjust the reference to the new superds
# if we had one ref before, we should still have exactly one
assert len(superdss) <= 1
dataset = list(superdss.keys())[0]
refds_path = dataset
elif refds_path:
# there is a single superdataset
superdss = {
refds_path: unique([ap['parentds']
for ap in to_process if 'parentds' in ap])}
else:
# sort all datasets under their potential superdatasets
# start from the top to get all subdatasets down the line
# and collate them into as few superdatasets as possible
# this is quick, just string operations
superdss = get_tree_roots(
unique([ap['parentds'] for ap in to_process if 'parentds' in ap]))
# for each "superdataset" check the tree of subdatasets and make sure
# we gather all datasets between the super and any subdataset
# so we can save them all bottom-up in order to be able to properly
# save the superdataset
# if this is called from e.g. `add` this is actually not necessary,
# but in the general case we cannot avoid it
# TODO maybe introduce a switch?
discovered = {}
for superds_path in superdss:
target_subs = superdss[superds_path]
discover_dataset_trace_to_targets(
# from here
superds_path,
# to all
target_subs,
[],
discovered)
# create a new minimally annotated path for each discovered dataset
discovered_added = set()
for parentds in discovered:
for subds in discovered[parentds]:
to_process.append(dict(
path=subds,
parentds=parentds,
type='dataset'))
discovered_added.add(subds)
# make sure we have an entry for each dataset, including those
# tha are just parents
for parentds in discovered:
if parentds not in discovered_added:
to_process.append(dict(
path=parentds,
type='dataset',
# make sure we save content of superds later on
process_content=True,
# but not do nasty things, like adding untracked content
# just because we discovered this dataset
process_updated_only=True))
# now re-annotate all paths, this will be fast for already annotated ones
# and will amend the annotation for others, deduplication happens here too
annotated_paths = AnnotatePaths.__call__(
path=to_process,
dataset=dataset,
# never recursion, done already
recursive=False,
action='save',
unavailable_path_status='',
nondataset_path_status='impossible',
return_type='generator',
# if there is an error now, we made this mistake in here
on_failure='stop')
# now sort into datasets so we can process them one by one
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
annotated_paths,
refds_path=refds_path)
assert(not completed)
# iterate over all datasets, starting at the bottom
for dspath in sorted(content_by_ds.keys(), reverse=True):
ds = Dataset(dspath)
res = get_status_dict('save', ds=ds, logger=lgr)
if not ds.is_installed():
# TODO This is likely impossible now
res['status'] = 'impossible'
res['message'] = ('dataset %s is not installed', ds)
yield res
continue
saved_state = save_dataset(
ds,
content_by_ds[dspath],
message=message)
res['status'] = 'ok' if saved_state else 'notneeded'
# MIH: let's tag even if there was nothing commit. I'd forget this
# option too often...
if version_tag:
try:
# TODO: check whether comment below is still true after
# removing the log swallowing:
# again cannot help but force-silence low-level code, because
# it screams like a made man instead of allowing top-level
# code an orderly error report
ds.repo.tag(version_tag)
# even if we haven't saved anything
res['status'] = 'ok'
yield res
except CommandError as e:
if saved_state:
# first we yield the result for the actual save
yield res
# and now complain that tagging didn't work
yield get_status_dict(
'save',
ds=ds,
logger=lgr,
status='error',
message=(
'cannot tag this version: %s',
e.stderr.strip()))
else:
yield res
def __call__(
path=None,
message=None,
dataset=None,
version_tag=None,
recursive=False,
recursion_limit=None,
updated=False,
message_file=None,
to_git=None,
jobs=None,
amend=False,
):
if message and message_file:
raise ValueError(
"Both a message and message file were specified for save()")
if amend and recursive:
raise ValueError("Cannot amend a commit recursively.")
path = ensure_list(path)
if message_file:
with open(message_file) as mfh:
message = mfh.read()
# we want 'normal' to achieve the most compact argument list
# for git calls
# untracked_mode = 'no' if updated else 'normal'
# TODO however, Repo.add() would refuse to add any dotfiles
# in a directory that is itself untracked, hence the only
# choice is to go with potentially crazy long lists
# until https://github.com/datalad/datalad/issues/1454
# has a resolution
untracked_mode = 'no' if updated else 'all'
# there are three basic scenarios:
# 1. save modifications to any already tracked content
# 2. save any content (including removal of deleted content)
# to bring things to a clean state
# 3. like (2), but only operate on a given subset of content
# identified by paths
# - all three have to work in conjunction with --recursive
# - the difference between (1) and (2) should be no more
# that a switch from --untracked=no to --untracked=all
# in Repo.save()
# we do not support
# - simultaneous operations on multiple datasets from disjoint
# dataset hierarchies, hence a single reference dataset must be
# identifiable from the either
# - curdir or
# - the `dataset` argument.
# This avoids complex annotation loops and hierarchy tracking.
# - any modification upwards from the root dataset
ds = require_dataset(dataset, check_installed=True, purpose='saving')
# use status() to do all discovery and annotation of paths
paths_by_ds = {}
for s in Status()(
# ATTN: it is vital to pass the `dataset` argument as it,
# and not a dataset instance in order to maintain the path
# semantics between here and the status() call
dataset=dataset,
path=path,
untracked=untracked_mode,
report_filetype=False,
recursive=recursive,
recursion_limit=recursion_limit,
on_failure='ignore',
# for save without recursion only commit matters
eval_subdataset_state='full' if recursive else 'commit',
result_renderer='disabled'):
if s['status'] == 'error':
# Downstream code can't do anything with these. Let the caller
# decide their fate.
yield s
continue
# fish out status dict for this parent dataset
ds_status = paths_by_ds.get(s['parentds'], {})
# reassemble path status info as repo.status() would have made it
ds_status[ut.Path(s['path'])] = \
{k: v for k, v in s.items()
if k not in (
'path', 'parentds', 'refds', 'status', 'action',
'logger')}
paths_by_ds[s['parentds']] = ds_status
lgr.debug('Determined %i datasets for saving from input arguments',
len(paths_by_ds))
# figure out what datasets to process, start with the ones containing
# the paths that were given as arguments
discovered_datasets = list(paths_by_ds.keys())
if dataset:
# if a reference dataset was given we want to save all the way up
# to it, so let's throw it into the mix
discovered_datasets.append(ds.path)
# sort the datasets into (potentially) disjoint hierarchies,
# or a single one, if a reference dataset was given
dataset_hierarchies = get_tree_roots(discovered_datasets)
for rootds, children in dataset_hierarchies.items():
edges = {}
discover_dataset_trace_to_targets(rootds,
children, [],
edges,
includeds=children)
for superds, subdss in edges.items():
superds_status = paths_by_ds.get(superds, {})
for subds in subdss:
subds_path = ut.Path(subds)
sub_status = superds_status.get(subds_path, {})
if not (sub_status.get("state") == "clean"
and sub_status.get("type") == "dataset"):
# TODO actually start from an entry that may already
# exist in the status record
superds_status[subds_path] = dict(
# shot from the hip, some status config
# to trigger this specific super/sub
# relation to be saved
state='untracked',
type='dataset')
paths_by_ds[superds] = superds_status
def save_ds(args, version_tag=None):
pdspath, paths = args
pds = Dataset(pdspath)
pds_repo = pds.repo
# pop status for this dataset, we are not coming back to it
pds_status = {
# for handing over to the low-level code, we recode any
# path relative to the real repo location, this avoid
# cumbersome symlink handling without context in the
# lower levels
pds_repo.pathobj / p.relative_to(pdspath): props
for p, props in paths.items()
}
start_commit = pds_repo.get_hexsha()
if not all(p['state'] == 'clean' for p in pds_status.values()) or \
(amend and message):
for res in pds_repo.save_(
message=message,
# make sure to have the `path` arg be None, as we want
# to prevent and bypass any additional repo.status()
# calls
paths=None,
# prevent whining of GitRepo
git=True
if not hasattr(ds.repo, 'annexstatus') else to_git,
# we are supplying the full status already, do not
# detect anything else
untracked='no',
_status=pds_status,
amend=amend):
# TODO remove stringification when datalad-core can handle
# path objects, or when PY3.6 is the lowest supported
# version
for k in ('path', 'refds'):
if k in res:
res[k] = str(
# recode path back to dataset path anchor
pds.pathobj /
res[k].relative_to(pds_repo.pathobj))
yield res
# report on the dataset itself
dsres = dict(
action='save',
type='dataset',
path=pds.path,
refds=ds.path,
status='ok'
if start_commit != pds_repo.get_hexsha() else 'notneeded',
logger=lgr,
)
if not version_tag:
yield dsres
return
try:
# method requires str
version_tag = str(version_tag)
pds_repo.tag(version_tag)
dsres.update(status='ok', version_tag=version_tag)
yield dsres
except CommandError as e:
if dsres['status'] == 'ok':
# first we yield the result for the actual save
# TODO: we will get duplicate dataset/save record obscuring
# progress reporting. yoh thought to decouple "tag" from "save"
# messages but was worrying that original authors would disagree
yield dsres.copy()
# and now complain that tagging didn't work
dsres.update(status='error',
message=('cannot tag this version: %s',
e.stderr.strip()))
yield dsres
if not paths_by_ds:
# Special case: empty repo. There's either an empty commit only or
# none at all. An empty one we can amend otherwise there's nothing
# to do.
if amend and ds.repo.get_hexsha():
yield from save_ds((ds.pathobj, dict()),
version_tag=version_tag)
else:
yield dict(action='save',
type='dataset',
path=ds.path,
refds=ds.path,
status='notneeded',
logger=lgr)
return
# TODO: in principle logging could be improved to go not by a dataset
# but by path(s) within subdatasets. That should provide a bit better ETA
# and more "dynamic" feedback than jumpy datasets count.
# See addurls where it is implemented that way by providing agg and another
# log_filter
yield from ProducerConsumerProgressLog(
sorted(paths_by_ds.items(), key=lambda v: v[0], reverse=True),
partial(save_ds, version_tag=version_tag),
safe_to_consume=no_subds_in_futures,
producer_future_key=lambda ds_items: ds_items[0],
jobs=jobs,
log_filter=_log_filter_save_dataset,
unit="datasets",
lgr=lgr,
)
def __call__(message=None,
path=None,
dataset=None,
all_updated=True,
version_tag=None,
recursive=False,
recursion_limit=None,
super_datasets=False,
message_file=None):
if not dataset and not path:
# we got nothing at all -> save what is staged in the repo in "this" directory?
# make sure we don't treat this as a user-provided '.' argument
path = [{'path': abspath(curdir), 'raw_input': False}]
refds_path = Interface.get_refds_path(dataset)
if message and message_file:
yield get_status_dict(
'save',
status='error',
path=refds_path,
message="Both a message and message file were specified",
logger=lgr)
return
if message_file:
with open(message_file) as mfh:
message = mfh.read()
to_process = []
got_nothing = True
for ap in AnnotatePaths.__call__(
path=path,
dataset=refds_path,
recursive=recursive,
recursion_limit=recursion_limit,
action='save',
unavailable_path_status='impossible',
unavailable_path_msg="path does not exist: %s",
nondataset_path_status='impossible',
modified='HEAD' if not path and recursive else None,
return_type='generator',
on_failure='ignore'):
if ap.get('state',
None) == 'untracked' and not ap.get('raw_input', False):
# this path was found untracked, but not explicitly given to save
# we will silently ignore this
continue
got_nothing = False
# next check should not be done during annotation, as it is possibly expensive
# and not generally useful
if ap.get('status', None) == 'impossible' and \
ap.get('state', None) == 'absent' and \
ap.get('parentds', None):
# this is not here anymore, but it might actually have been a deleted
# component
if relpath(ap['path'], start=ap['parentds']) \
in Dataset(ap['parentds']).repo.get_deleted_files():
# ok, this is a staged deletion that we want to save
ap['status'] = ''
del ap['message']
if ap.get('status', None):
# this is done
yield ap
continue
# for things like: `ds.save()`
# or recursively discovered datasets
if ap['path'] == refds_path or \
(ap.get('type', None) == 'dataset' and
not ap.get('raw_input', False) and
not ap.get('state', None) == 'absent'):
ap['process_content'] = True
ap['process_updated_only'] = all_updated
to_process.append(ap)
lgr.log(2, "save, to_process=%r", to_process)
if got_nothing and recursive and refds_path:
# path annotation yielded nothing, most likely cause is that nothing
# was found modified, we need to say something about the reference
# dataset
yield get_status_dict('save',
status='notneeded',
path=refds_path,
type='dataset',
logger=lgr)
return
if not to_process:
# nothing left to do, potentially all errored before
return
if super_datasets:
# search for the topmost superdatasets of any path
dss = [
Dataset(ap.get('parentds', ap['path'])) for ap in to_process
]
superdss = [ds.get_superdataset(topmost=True) for ds in dss]
superdss = get_tree_roots(
unique(ds.path for ds in dss + superdss if ds))
if dataset:
# need to adjust the reference to the new superds
# if we had one ref before, we should still have exactly one
assert len(superdss) <= 1
dataset = list(superdss.keys())[0]
refds_path = dataset
elif refds_path:
# there is a single superdataset
superdss = {
refds_path:
unique(
[ap['parentds'] for ap in to_process if 'parentds' in ap])
}
else:
# sort all datasets under their potential superdatasets
# start from the top to get all subdatasets down the line
# and collate them into as few superdatasets as possible
# this is quick, just string operations
superdss = get_tree_roots(
unique(
[ap['parentds'] for ap in to_process if 'parentds' in ap]))
# for each "superdataset" check the tree of subdatasets and make sure
# we gather all datasets between the super and any subdataset
# so we can save them all bottom-up in order to be able to properly
# save the superdataset
# if this is called from e.g. `add` this is actually not necessary,
# but in the general case we cannot avoid it
# TODO maybe introduce a switch?
discovered = {}
for superds_path in superdss:
target_subs = superdss[superds_path]
discover_dataset_trace_to_targets(
# from here
superds_path,
# to all
target_subs,
[],
discovered)
# create a new minimally annotated path for each discovered dataset
discovered_added = set()
for parentds in discovered:
for subds in discovered[parentds]:
to_process.append(
dict(path=subds, parentds=parentds, type='dataset'))
discovered_added.add(subds)
# make sure we have an entry for each dataset, including those
# tha are just parents
for parentds in discovered:
if parentds not in discovered_added:
to_process.append(
dict(
path=parentds,
type='dataset',
# make sure we save content of superds later on
process_content=True,
# but not do nasty things, like adding untracked content
# just because we discovered this dataset
process_updated_only=True))
# now re-annotate all paths, this will be fast for already annotated ones
# and will amend the annotation for others, deduplication happens here too
annotated_paths = AnnotatePaths.__call__(
path=to_process,
dataset=dataset,
# never recursion, done already
recursive=False,
action='save',
unavailable_path_status='',
nondataset_path_status='impossible',
return_type='generator',
# if there is an error now, we made this mistake in here
on_failure='stop')
# now sort into datasets so we can process them one by one
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
annotated_paths,
refds_path=refds_path)
assert (not completed)
# iterate over all datasets, starting at the bottom
for dspath in sorted(content_by_ds.keys(), reverse=True):
ds = Dataset(dspath)
res = get_status_dict('save', ds=ds, logger=lgr)
if not ds.is_installed():
# TODO This is likely impossible now
res['status'] = 'impossible'
res['message'] = ('dataset %s is not installed', ds)
yield res
continue
saved_state = save_dataset(ds,
content_by_ds[dspath],
message=message)
res['status'] = 'ok' if saved_state else 'notneeded'
# MIH: let's tag even if there was nothing commit. I'd forget this
# option too often...
if version_tag:
try:
# TODO: check whether comment below is still true after
# removing the log swallowing:
# again cannot help but force-silence low-level code, because
# it screams like a made man instead of allowing top-level
# code an orderly error report
ds.repo.tag(version_tag)
# even if we haven't saved anything
res['status'] = 'ok'
yield res
except CommandError as e:
if saved_state:
# first we yield the result for the actual save
yield res
# and now complain that tagging didn't work
yield get_status_dict(
'save',
ds=ds,
logger=lgr,
status='error',
message=('cannot tag this version: %s',
e.stderr.strip()))
else:
yield res
def __call__(
path=None,
dataset=None,
recursive=False,
recursion_limit=None,
update_mode='target',
incremental=False,
force_extraction=False,
save=True):
refds_path = Interface.get_refds_path(dataset)
# it really doesn't work without a dataset
ds = require_dataset(
dataset, check_installed=True, purpose='metadata aggregation')
path = assure_list(path)
if not path:
# then current/reference dataset is "aggregated"
# We should not add ds.path always since then --recursive would
# also recurse current even if paths are given
path.append(ds.path)
agginfo_db_location, agg_base_path = get_ds_aggregate_db_locations(ds)
agginfo_db = load_ds_aggregate_db(ds, abspath=True)
to_save = []
to_aggregate = set()
for ap in AnnotatePaths.__call__(
dataset=refds_path,
path=path,
recursive=recursive,
recursion_limit=recursion_limit,
action='aggregate_metadata',
# uninstalled subdatasets could be queried via aggregated metadata
# -> no 'error'
unavailable_path_status='',
nondataset_path_status='error',
return_type='generator',
on_failure='ignore'):
if ap.get('status', None):
# this is done
yield ap
continue
ap_type = ap.get('type', None)
ap_state = ap.get('state', None)
assert('parentds' in ap or ap_type == 'dataset')
if ap_type == 'dataset' and ap_state != 'absent':
# a present dataset, we can take directly from it
aggsrc = ap['path']
lgr.info('Aggregate metadata for dataset %s', aggsrc)
else:
# everything else needs to come from the parent
aggsrc = ap['parentds']
if ap_state == 'absent':
lgr.info(
'Attempt to use pre-aggregate metadata for absent %s from dataset at %s',
ap['path'],
aggsrc)
else:
lgr.info(
'Aggregate metadata for %s from dataset at %s',
ap['path'],
aggsrc)
to_aggregate.add(aggsrc)
if ap_state == 'absent':
# key thought: recursive is done by path annotation, hence
# once we hit an absent dataset, we are 100% certain that
# there is nothing to recurse into on the file system
# hence we only have to look into the aggregated metadata
# of the last available dataset in the dataset tree edge
#
# if there is nothing at this path, we need to look into the
# parentds and check if we know anything about this path
# if we do, we need to grab all the info and objects
# if not, we need to error
res = _get_dsinfo_from_aggmetadata(
aggsrc, ap['path'], recursive, agginfo_db)
if not isinstance(res, list):
yield get_status_dict(
status='impossible',
message=res,
action='aggregate_metadata',
path=ap['path'],
logger=lgr)
continue
# cue for aggregation
to_aggregate.update(res)
else:
# actually aggregate metadata for this dataset, immediately place
# generated objects into the aggregated or reference dataset,
# and put info into DB to get the distributed to all datasets
# that need to be updated
errored = _dump_extracted_metadata(
ds,
Dataset(aggsrc),
agginfo_db,
to_save,
force_extraction,
agg_base_path)
if errored:
yield get_status_dict(
status='error',
message='Metadata extraction failed (see previous error message, set datalad.runtime.raiseonerror=yes to fail immediately)',
action='aggregate_metadata',
path=aggsrc,
logger=lgr)
# at this point we have dumped all aggregated metadata into object files
# somewhere, we know what needs saving, but having saved anything, and
# we know about the states of all aggregated dataset in the DB
# what remains to do is to update all dataset, so they have there own copy
# of aggregated metadata and update their respective aggregate.json with
# info on what states we just aggregated from
# first, let's figure out what dataset need updating at all
# get adjencency info of the dataset tree spanning the base to all leaf dataset
# associated with the path arguments
if update_mode == 'all':
ds_adj = {}
discover_dataset_trace_to_targets(
ds.path, to_aggregate, [], ds_adj,
# we know that to_aggregate only lists datasets, existing and
# absent ones -- we want to aggregate all of them, either from
# just extracted metadata, or from previously aggregated metadata
# of the closest superdataset
includeds=to_aggregate)
# TODO we need to work in the info about dataset that we only got from
# aggregated metadata, that had no trace on the file system in here!!
subtrees = _adj2subtrees(ds.path, ds_adj, to_aggregate)
elif update_mode == 'target':
subtrees = {ds.path: list(agginfo_db.keys())}
else:
raise ValueError(
"unknown `update_mode` '%s' for metadata aggregation", update_mode)
# go over datasets in bottom-up fashion
for parentds_path in sorted(subtrees, reverse=True):
lgr.info('Update aggregate metadata in dataset at: %s', parentds_path)
_update_ds_agginfo(
ds.path,
parentds_path,
subtrees[parentds_path],
incremental,
agginfo_db,
to_save)
# update complete
res = get_status_dict(
status='ok',
action='aggregate_metadata',
path=parentds_path,
type='dataset',
logger=lgr)
res.update(agginfo_db.get(parentds_path, {}))
yield res
#
# save potential modifications to dataset global metadata
#
if not to_save:
return
lgr.info('Attempting to save %i files/datasets', len(to_save))
for res in Save.__call__(
path=to_save,
dataset=refds_path,
message='[DATALAD] Dataset aggregate metadata update',
return_type='generator',
result_xfm=None,
result_filter=None,
on_failure='ignore'):
yield res
