def __call__(path=None,
dataset=None,
fulfilled=None,
recursive=False,
recursion_limit=None,
contains=None,
bottomup=False,
set_property=None,
delete_property=None):
# no constraints given -> query subdatasets under curdir
if not path and dataset is None:
path = os.curdir
paths = [rev_resolve_path(p, dataset) for p in assure_list(path)] \
if path else None
ds = require_dataset(dataset,
check_installed=False,
purpose='subdataset reporting/modification')
lgr.debug('Query subdatasets of %s', dataset)
if paths is not None:
lgr.debug('Query subdatasets underneath paths: %s', paths)
refds_path = ds.path
# XXX this seems strange, but is tested to be the case -- I'd rather set
# `check_installed` to true above and fail
if not GitRepo.is_valid_repo(refds_path):
return
# return as quickly as possible
if isinstance(recursion_limit, int) and (recursion_limit <= 0):
return
if set_property:
for k, v in set_property:
if valid_key.match(k) is None:
raise ValueError(
"key '%s' is invalid (alphanumeric plus '-' only, must "
"start with a letter)" % k)
if contains:
contains = [
rev_resolve_path(c, dataset) for c in assure_list(contains)
]
for r in _get_submodules(ds, paths, fulfilled, recursive,
recursion_limit, contains, bottomup,
set_property, delete_property, refds_path):
# a boat-load of ancient code consumes this and is ignorant of
# Path objects
r['path'] = text_type(r['path'])
# without the refds_path cannot be rendered/converted relative
# in the eval_results decorator
r['refds'] = refds_path
yield r
def __call__(
path=None,
dataset=None,
fulfilled=None,
recursive=False,
recursion_limit=None,
contains=None,
bottomup=False,
set_property=None,
delete_property=None):
# no constraints given -> query subdatasets under curdir
if not path and dataset is None:
path = os.curdir
paths = [rev_resolve_path(p, dataset) for p in assure_list(path)] \
if path else None
ds = require_dataset(
dataset, check_installed=False, purpose='subdataset reporting/modification')
lgr.debug('Query subdatasets of %s', dataset)
if paths is not None:
lgr.debug('Query subdatasets underneath paths: %s', paths)
refds_path = ds.path
# XXX this seems strange, but is tested to be the case -- I'd rather set
# `check_installed` to true above and fail
if not GitRepo.is_valid_repo(refds_path):
return
# return as quickly as possible
if isinstance(recursion_limit, int) and (recursion_limit <= 0):
return
if set_property:
for k, v in set_property:
if valid_key.match(k) is None:
raise ValueError(
"key '%s' is invalid (alphanumeric plus '-' only, must "
"start with a letter)" % k)
if contains:
contains = [rev_resolve_path(c, dataset) for c in assure_list(contains)]
for r in _get_submodules(
ds, paths, fulfilled, recursive, recursion_limit,
contains, bottomup, set_property, delete_property,
refds_path):
# a boat-load of ancient code consumes this and is ignorant of
# Path objects
r['path'] = text_type(r['path'])
# without the refds_path cannot be rendered/converted relative
# in the eval_results decorator
r['refds'] = refds_path
yield r
def __call__(path=None,
initopts=None,
force=False,
description=None,
dataset=None,
no_annex=False,
fake_dates=False,
cfg_proc=None):
refds_path = dataset.path if hasattr(dataset, 'path') else dataset
# two major cases
# 1. we got a `dataset` -> we either want to create it (path is None),
# or another dataset in it (path is not None)
# 2. we got no dataset -> we want to create a fresh dataset at the
# desired location, either at `path` or PWD
# sanity check first
if no_annex:
if description:
raise ValueError("Incompatible arguments: cannot specify "
"description for annex repo and declaring "
"no annex repo.")
if path:
path = rev_resolve_path(path, dataset)
path = path if path \
else getpwd() if dataset is None \
else refds_path
# we know that we need to create a dataset at `path`
assert (path is not None)
# prep for yield
res = dict(action='create',
path=text_type(path),
logger=lgr,
type='dataset',
refds=refds_path)
refds = None
if refds_path and refds_path != path:
refds = require_dataset(refds_path,
check_installed=True,
purpose='creating a subdataset')
path_inrefds = path_under_rev_dataset(refds, path)
if path_inrefds is None:
yield dict(
res,
status='error',
message=(
"dataset containing given paths is not underneath "
"the reference dataset %s: %s", dataset,
text_type(path)),
)
return
# try to locate an immediate parent dataset
# we want to know this (irrespective of whether we plan on adding
# this new dataset to a parent) in order to avoid conflicts with
# a potentially absent/uninstalled subdataset of the parent
# in this location
# it will cost some filesystem traversal though...
parentds_path = rev_get_dataset_root(
op.normpath(op.join(text_type(path), os.pardir)))
if parentds_path:
prepo = GitRepo(parentds_path)
parentds_path = ut.Path(parentds_path)
# we cannot get away with a simple
# GitRepo.get_content_info(), as we need to detect
# uninstalled/added subdatasets too
check_path = ut.Path(path)
pstatus = prepo.status(
untracked='no',
# limit query to target path for a potentially massive speed-up
paths=[check_path.relative_to(parentds_path)])
if any(check_path == p or check_path in p.parents
for p in pstatus):
# redo the check in a slower fashion, it is already broken
# let's take our time for a proper error message
conflict = [
p for p in pstatus
if check_path == p or check_path in p.parents
]
res.update({
'status':
'error',
'message':
('collision with content in parent dataset at %s: %s',
text_type(parentds_path),
[text_type(c) for c in conflict])
})
yield res
return
# another set of check to see whether the target path is pointing
# into a known subdataset that is not around ATM
subds_status = {
parentds_path / k.relative_to(prepo.path)
for k, v in iteritems(pstatus)
if v.get('type', None) == 'dataset'
}
check_paths = [check_path]
check_paths.extend(check_path.parents)
if any(p in subds_status for p in check_paths):
conflict = [p for p in check_paths if p in subds_status]
res.update({
'status':
'error',
'message':
('collision with %s (dataset) in dataset %s',
text_type(conflict[0]), text_type(parentds_path))
})
yield res
return
# important to use the given Dataset object to avoid spurious ID
# changes with not-yet-materialized Datasets
tbds = dataset if isinstance(dataset, Dataset) and \
dataset.path == path else Dataset(text_type(path))
# don't create in non-empty directory without `force`:
if op.isdir(tbds.path) and listdir(tbds.path) != [] and not force:
res.update({
'status':
'error',
'message':
'will not create a dataset in a non-empty directory, use '
'`force` option to ignore'
})
yield res
return
# stuff that we create and want to have tracked with git (not annex)
add_to_git = {}
if initopts is not None and isinstance(initopts, list):
initopts = {'_from_cmdline_': initopts}
# create and configure desired repository
if no_annex:
lgr.info("Creating a new git repo at %s", tbds.path)
tbrepo = GitRepo(tbds.path,
url=None,
create=True,
create_sanity_checks=False,
git_opts=initopts,
fake_dates=fake_dates)
# place a .noannex file to indicate annex to leave this repo alone
stamp_path = ut.Path(tbrepo.path) / '.noannex'
stamp_path.touch()
add_to_git[stamp_path] = {'type': 'file', 'state': 'untracked'}
else:
# always come with annex when created from scratch
lgr.info("Creating a new annex repo at %s", tbds.path)
tbrepo = AnnexRepo(
tbds.path,
url=None,
create=True,
create_sanity_checks=False,
# do not set backend here, to avoid a dedicated commit
backend=None,
# None causes version to be taken from config
version=None,
description=description,
git_opts=initopts,
fake_dates=fake_dates)
# set the annex backend in .gitattributes as a staged change
tbrepo.set_default_backend(cfg.obtain('datalad.repo.backend'),
persistent=True,
commit=False)
add_to_git[tbds.repo.pathobj / '.gitattributes'] = {
'type': 'file',
'state': 'added'
}
# make sure that v6 annex repos never commit content under .datalad
attrs_cfg = (('config', 'annex.largefiles', 'nothing'), (
'metadata/aggregate*', 'annex.largefiles', 'nothing'
), ('metadata/objects/**', 'annex.largefiles', '({})'.format(
cfg.obtain('datalad.metadata.create-aggregate-annex-limit'))))
attrs = tbds.repo.get_gitattributes(
[op.join('.datalad', i[0]) for i in attrs_cfg])
set_attrs = []
for p, k, v in attrs_cfg:
if not attrs.get(op.join('.datalad', p), {}).get(k, None) == v:
set_attrs.append((p, {k: v}))
if set_attrs:
tbds.repo.set_gitattributes(set_attrs,
attrfile=op.join(
'.datalad', '.gitattributes'))
# prevent git annex from ever annexing .git* stuff (gh-1597)
attrs = tbds.repo.get_gitattributes('.git')
if not attrs.get('.git', {}).get('annex.largefiles',
None) == 'nothing':
tbds.repo.set_gitattributes([('**/.git*', {
'annex.largefiles': 'nothing'
})])
# must use the repo.pathobj as this will have resolved symlinks
add_to_git[tbds.repo.pathobj / '.gitattributes'] = {
'type': 'file',
'state': 'untracked'
}
# record an ID for this repo for the afterlife
# to be able to track siblings and children
id_var = 'datalad.dataset.id'
# Note, that Dataset property `id` will change when we unset the
# respective config. Therefore store it before:
tbds_id = tbds.id
if id_var in tbds.config:
# make sure we reset this variable completely, in case of a
# re-create
tbds.config.unset(id_var, where='dataset')
if _seed is None:
# just the standard way
uuid_id = uuid.uuid1().urn.split(':')[-1]
else:
# Let's generate preseeded ones
uuid_id = str(uuid.UUID(int=random.getrandbits(128)))
tbds.config.add(id_var,
tbds_id if tbds_id is not None else uuid_id,
where='dataset',
reload=False)
# make config overrides permanent in the repo config
# this is similar to what `annex init` does
# we are only doing this for config overrides and do not expose
# a dedicated argument, because it is sufficient for the cmdline
# and unnecessary for the Python API (there could simply be a
# subsequence ds.config.add() call)
for k, v in iteritems(tbds.config.overrides):
tbds.config.add(k, v, where='local', reload=False)
# all config manipulation is done -> fll reload
tbds.config.reload()
# must use the repo.pathobj as this will have resolved symlinks
add_to_git[tbds.repo.pathobj / '.datalad'] = {
'type': 'directory',
'state': 'untracked'
}
# save everything, we need to do this now and cannot merge with the
# call below, because we may need to add this subdataset to a parent
# but cannot until we have a first commit
tbds.repo.save(
message='[DATALAD] new dataset',
git=True,
# we have to supply our own custom status, as the repo does
# not have a single commit yet and the is no HEAD reference
# TODO make `GitRepo.status()` robust to this state.
_status=add_to_git,
)
# the next only makes sense if we saved the created dataset,
# otherwise we have no committed state to be registered
# in the parent
if isinstance(refds, Dataset) and refds.path != tbds.path:
# we created a dataset in another dataset
# -> make submodule
for r in refds.save(path=tbds.path, ):
yield r
res.update({'status': 'ok'})
yield res
for cfg_proc_ in cfg_proc or []:
for r in tbds.run_procedure('cfg_' + cfg_proc_):
yield r
def sort_paths_by_datasets(orig_dataset_arg, paths):
"""Sort paths into actually present datasets
Parameters
----------
orig_dataset_arg : None or str
The original dataset argument of the calling command. This is
used to determine the path specification semantics, i.e.
relative to CWD vs. relative to a given dataset
paths : list
Paths as given to the calling command
Returns
-------
OrderedDict, list
The dictionary contains all to-be-sorted paths as values to
their respective containing datasets paths (as keys). The second
list contains status dicts for any errors that may have occurred
during processing. They can be yielded in the context of
the calling command.
"""
errors = []
paths_by_ds = OrderedDict()
# sort any path argument into the respective subdatasets
for p in sorted(paths):
# it is important to capture the exact form of the
# given path argument, before any normalization happens
# for further decision logic below
orig_path = text_type(p)
p = rev_resolve_path(p, orig_dataset_arg)
root = rev_get_dataset_root(text_type(p))
if root is None:
# no root, not possibly underneath the refds
errors.append(
dict(action='status',
path=p,
status='error',
message='path not underneath this dataset',
logger=lgr))
continue
else:
if orig_dataset_arg and root == text_type(p) and \
not orig_path.endswith(op.sep):
# the given path is pointing to a dataset
# distinguish rsync-link syntax to identify
# the dataset as whole (e.g. 'ds') vs its
# content (e.g. 'ds/')
super_root = rev_get_dataset_root(op.dirname(root))
if super_root:
# the dataset identified by the path argument
# is contained in a superdataset, and no
# trailing path separator was found in the
# argument -> user wants to address the dataset
# as a whole (in the superdataset)
root = super_root
root = ut.Path(root)
ps = paths_by_ds.get(root, [])
ps.append(p)
paths_by_ds[root] = ps
return paths_by_ds, errors
def __call__(path=None,
dataset=None,
annex=None,
untracked='normal',
recursive=False,
recursion_limit=None,
eval_subdataset_state='full'):
# To the next white knight that comes in to re-implement `status` as a
# special case of `diff`. There is one fundamental difference between
# the two commands: `status` can always use the worktree as evident on
# disk as a contraint (e.g. to figure out which subdataset a path is in)
# `diff` cannot do that (everything need to be handled based on a
# "virtual" representation of a dataset hierarchy).
# MIH concludes that while `status` can be implemented as a special case
# of `diff` doing so would complicate and slow down both `diff` and
# `status`. So while the apparent almost code-duplication between the
# two commands feels wrong, the benefit is speed. Any future RF should
# come with evidence that speed does not suffer, and complexity stays
# on a manageable level
ds = require_dataset(dataset,
check_installed=True,
purpose='status reporting')
paths_by_ds = OrderedDict()
if path:
# sort any path argument into the respective subdatasets
for p in sorted(assure_list(path)):
# it is important to capture the exact form of the
# given path argument, before any normalization happens
# for further decision logic below
orig_path = text_type(p)
p = rev_resolve_path(p, dataset)
root = rev_get_dataset_root(text_type(p))
if root is None:
# no root, not possibly underneath the refds
yield dict(action='status',
path=p,
refds=ds.path,
status='error',
message='path not underneath this dataset',
logger=lgr)
continue
else:
if dataset and root == text_type(p) and \
not (orig_path.endswith(op.sep) or
orig_path == "."):
# the given path is pointing to a dataset
# distinguish rsync-link syntax to identify
# the dataset as whole (e.g. 'ds') vs its
# content (e.g. 'ds/')
super_root = rev_get_dataset_root(op.dirname(root))
if super_root:
# the dataset identified by the path argument
# is contained in a superdataset, and no
# trailing path separator was found in the
# argument -> user wants to address the dataset
# as a whole (in the superdataset)
root = super_root
root = ut.Path(root)
ps = paths_by_ds.get(root, [])
ps.append(p)
paths_by_ds[root] = ps
else:
paths_by_ds[ds.pathobj] = None
queried = set()
content_info_cache = {}
while paths_by_ds:
qdspath, qpaths = paths_by_ds.popitem(last=False)
if qpaths and qdspath in qpaths:
# this is supposed to be a full query, save some
# cycles sifting through the actual path arguments
qpaths = []
# try to recode the dataset path wrt to the reference
# dataset
# the path that it might have been located by could
# have been a resolved path or another funky thing
qds_inrefds = path_under_rev_dataset(ds, qdspath)
if qds_inrefds is None:
# nothing we support handling any further
# there is only a single refds
yield dict(
path=text_type(qdspath),
refds=ds.path,
action='status',
status='error',
message=(
"dataset containing given paths is not underneath "
"the reference dataset %s: %s", ds, qpaths),
logger=lgr,
)
continue
elif qds_inrefds != qdspath:
# the path this dataset was located by is not how it would
# be referenced underneath the refds (possibly resolved
# realpath) -> recode all paths to be underneath the refds
qpaths = [qds_inrefds / p.relative_to(qdspath) for p in qpaths]
qdspath = qds_inrefds
if qdspath in queried:
# do not report on a single dataset twice
continue
qds = Dataset(text_type(qdspath))
for r in _yield_status(
qds, qpaths, annex, untracked, recursion_limit
if recursion_limit is not None else -1 if recursive else 0,
queried, eval_subdataset_state, content_info_cache):
yield dict(
r,
refds=ds.path,
action='status',
status='ok',
)
def __call__(path=None,
dataset=None,
recursive=False,
recursion_limit=None):
refds = require_dataset(dataset,
check_installed=True,
purpose="unlocking")
# Before passing the results to status()
# * record explicitly specified non-directory paths so that we can
# decide whether to yield a result for reported paths
# * filter out and yield results for paths that don't exist
paths_nondir = set()
paths_lexist = None
if path:
path = rev_resolve_path(assure_list(path), ds=dataset)
paths_lexist = []
for p in path:
if p.exists() or p.is_symlink():
paths_lexist.append(p)
if not p.is_dir():
paths_nondir.add(p)
res_kwargs = dict(action='unlock', logger=lgr, refds=refds.path)
if path:
for p in set(path).difference(set(paths_lexist)):
yield get_status_dict(status="impossible",
path=text_type(p),
type="file",
message="path does not exist",
**res_kwargs)
if not (paths_lexist or paths_lexist is None):
return
# Collect information on the paths to unlock.
to_unlock = defaultdict(list) # ds => paths (relative to ds)
for res in Status()(
# ATTN: it is vital to pass the `dataset` argument as it,
# and not a dataset instance in order to maintain the path
# semantics between here and the status() call
dataset=dataset,
path=paths_lexist,
untracked="normal" if paths_nondir else "no",
annex="availability",
recursive=recursive,
recursion_limit=recursion_limit,
result_renderer='disabled',
on_failure="ignore"):
if res["action"] != "status" or res["status"] != "ok":
yield res
continue
has_content = res.get("has_content")
if has_content:
parentds = res["parentds"]
to_unlock[parentds].append(op.relpath(res["path"], parentds))
elif paths_nondir and Path(res["path"]) in paths_nondir:
if has_content is False:
msg = "no content present"
status = "impossible"
elif res["state"] == "untracked":
msg = "untracked"
status = "impossible"
else:
# This is either a regular git file or an unlocked annex
# file.
msg = "non-annex file"
status = "notneeded"
yield get_status_dict(status=status,
path=res["path"],
type="file",
message="{}; cannot unlock".format(msg),
**res_kwargs)
# Do the actual unlocking.
for ds_path, files in to_unlock.items():
ds = Dataset(ds_path)
for r in ds.repo.unlock(files):
yield get_status_dict(path=op.join(ds.path, r),
status='ok',
type='file',
**res_kwargs)
def __call__(
path=None,
dataset=None,
annex=None,
untracked='normal',
recursive=False,
recursion_limit=None,
eval_subdataset_state='full'):
# To the next white knight that comes in to re-implement `status` as a
# special case of `diff`. There is one fundamental difference between
# the two commands: `status` can always use the worktree as evident on
# disk as a contraint (e.g. to figure out which subdataset a path is in)
# `diff` cannot do that (everything need to be handled based on a
# "virtual" representation of a dataset hierarchy).
# MIH concludes that while `status` can be implemented as a special case
# of `diff` doing so would complicate and slow down both `diff` and
# `status`. So while the apparent almost code-duplication between the
# two commands feels wrong, the benefit is speed. Any future RF should
# come with evidence that speed does not suffer, and complexity stays
# on a manageable level
ds = require_dataset(
dataset, check_installed=True, purpose='status reporting')
paths_by_ds = OrderedDict()
if path:
# sort any path argument into the respective subdatasets
for p in sorted(assure_list(path)):
# it is important to capture the exact form of the
# given path argument, before any normalization happens
# for further decision logic below
orig_path = text_type(p)
p = rev_resolve_path(p, dataset)
root = rev_get_dataset_root(text_type(p))
if root is None:
# no root, not possibly underneath the refds
yield dict(
action='status',
path=p,
refds=ds.path,
status='error',
message='path not underneath this dataset',
logger=lgr)
continue
else:
if dataset and root == text_type(p) and \
not (orig_path.endswith(op.sep) or
orig_path == "."):
# the given path is pointing to a dataset
# distinguish rsync-link syntax to identify
# the dataset as whole (e.g. 'ds') vs its
# content (e.g. 'ds/')
super_root = rev_get_dataset_root(op.dirname(root))
if super_root:
# the dataset identified by the path argument
# is contained in a superdataset, and no
# trailing path separator was found in the
# argument -> user wants to address the dataset
# as a whole (in the superdataset)
root = super_root
root = ut.Path(root)
ps = paths_by_ds.get(root, [])
ps.append(p)
paths_by_ds[root] = ps
else:
paths_by_ds[ds.pathobj] = None
queried = set()
content_info_cache = {}
while paths_by_ds:
qdspath, qpaths = paths_by_ds.popitem(last=False)
if qpaths and qdspath in qpaths:
# this is supposed to be a full query, save some
# cycles sifting through the actual path arguments
qpaths = []
# try to recode the dataset path wrt to the reference
# dataset
# the path that it might have been located by could
# have been a resolved path or another funky thing
qds_inrefds = path_under_rev_dataset(ds, qdspath)
if qds_inrefds is None:
# nothing we support handling any further
# there is only a single refds
yield dict(
path=text_type(qdspath),
refds=ds.path,
action='status',
status='error',
message=(
"dataset containing given paths is not underneath "
"the reference dataset %s: %s",
ds, qpaths),
logger=lgr,
)
continue
elif qds_inrefds != qdspath:
# the path this dataset was located by is not how it would
# be referenced underneath the refds (possibly resolved
# realpath) -> recode all paths to be underneath the refds
qpaths = [qds_inrefds / p.relative_to(qdspath) for p in qpaths]
qdspath = qds_inrefds
if qdspath in queried:
# do not report on a single dataset twice
continue
qds = Dataset(text_type(qdspath))
for r in _yield_status(
qds,
qpaths,
annex,
untracked,
recursion_limit
if recursion_limit is not None else -1
if recursive else 0,
queried,
eval_subdataset_state,
content_info_cache):
yield dict(
r,
refds=ds.path,
action='status',
status='ok',
)
def __call__(
path=None,
dataset=None,
reporton='all',
recursive=False):
# prep results
res_kwargs = dict(action='meta_dump', logger=lgr)
ds = require_dataset(
dataset=dataset,
check_installed=True,
purpose='aggregate metadata query')
if dataset:
res_kwargs['refds'] = ds.path
agginfos = get_ds_aggregate_db(
ds.pathobj,
version=str(aggregate_layout_version),
# we are handling errors below
warn_absent=False,
)
if not agginfos:
# if there has ever been an aggregation run, this file would
# exist, hence there has not been and we need to tell this
# to people
yield get_status_dict(
ds=ds,
status='impossible',
message='metadata aggregation has never been performed in '
'this dataset',
**res_kwargs)
return
if not path:
# implement https://github.com/datalad/datalad/issues/3282
path = ds.pathobj if isinstance(dataset, Dataset) else os.getcwd()
# check for paths that are not underneath this dataset
resolved_paths = set()
for p in assure_list(path):
p = rev_resolve_path(p, dataset)
if p != ds.pathobj and ds.pathobj not in p.parents:
raise ValueError(
'given path {} is not underneath dataset {}'.format(
p, ds))
resolved_paths.add(p)
# sort paths into their containing dataset aggregate records
paths_by_ds = {}
while resolved_paths:
resolved_path = resolved_paths.pop()
# find the first dataset that matches
for aggdspath in sorted(agginfos, reverse=True):
if recursive and resolved_path in aggdspath.parents:
ps = paths_by_ds.get(aggdspath, set())
ps.add(aggdspath)
paths_by_ds[aggdspath] = ps
elif aggdspath == resolved_path \
or aggdspath in resolved_path.parents:
ps = paths_by_ds.get(aggdspath, set())
ps.add(resolved_path)
paths_by_ds[aggdspath] = ps
# stop when the containing dataset is found
break
# which files do we need to have locally to perform the query
info_keys = \
('dataset_info', 'content_info') \
if reporton in ('all', 'jsonld') else \
('dataset_info',) if reporton == 'datasets' else \
('content_info',) if reporton == 'files' else \
[]
objfiles = [
text_type(agginfos[d][t])
for d in paths_by_ds
for t in info_keys
if t in agginfos[d]
]
lgr.debug(
'Verifying/achieving local availability of %i metadata objects',
len(objfiles))
if objfiles:
for r in ds.get(
path=objfiles,
result_renderer='disabled',
return_type='generator'):
# report only of not a success as this is an internal operation
# that a user would not (need to) expect
if success_status_map.get(r['status'], False) != 'success': # pragma: no cover
yield r
contexts = {}
nodes_by_context = {}
parentds = []
# loop over all records to get complete parentds relationships
for aggdspath in sorted(agginfos):
while parentds and parentds[-1] not in aggdspath.parents:
parentds.pop()
if aggdspath not in paths_by_ds:
# nothing to say about this
parentds.append(aggdspath)
continue
agg_record = agginfos[aggdspath]
if reporton == 'aggregates':
# we do not need to loop over the actual query paths, as
# the aggregates of the containing dataset will contain
# the desired info, if any exists
# convert pathobj before emitting until we became more clever
info = {k: text_type(v) if isinstance(v, ut.PurePath) else v
for k, v in iteritems(agg_record)}
info.update(
path=text_type(aggdspath),
type='dataset',
)
if aggdspath == ds.pathobj:
info['layout_version'] = aggregate_layout_version
if parentds:
info['parentds'] = text_type(parentds[-1])
yield dict(
info,
status='ok',
**res_kwargs
)
parentds.append(aggdspath)
continue
# pull out actual metadata records
for res in _yield_metadata_records(
aggdspath,
agg_record,
paths_by_ds[aggdspath],
reporton,
parentds=parentds[-1] if parentds else None):
if reporton != 'jsonld':
yield dict(
res,
**res_kwargs
)
continue
collect_jsonld_metadata(
aggdspath, res, nodes_by_context, contexts)
parentds.append(aggdspath)
if reporton == 'jsonld':
yield dict(
status='ok',
type='dataset',
path=ds.path,
metadata=format_jsonld_metadata(nodes_by_context),
refcommit=agginfos[ds.pathobj]['refcommit'],
**res_kwargs)
def __call__(
path=None,
initopts=None,
force=False,
description=None,
dataset=None,
no_annex=False,
fake_dates=False,
cfg_proc=None
):
refds_path = dataset.path if hasattr(dataset, 'path') else dataset
# two major cases
# 1. we got a `dataset` -> we either want to create it (path is None),
# or another dataset in it (path is not None)
# 2. we got no dataset -> we want to create a fresh dataset at the
# desired location, either at `path` or PWD
# sanity check first
if no_annex:
if description:
raise ValueError("Incompatible arguments: cannot specify "
"description for annex repo and declaring "
"no annex repo.")
if path:
path = rev_resolve_path(path, dataset)
path = path if path \
else getpwd() if dataset is None \
else refds_path
# we know that we need to create a dataset at `path`
assert(path is not None)
# prep for yield
res = dict(action='create', path=text_type(path),
logger=lgr, type='dataset',
refds=refds_path)
refds = None
if refds_path and refds_path != path:
refds = require_dataset(
refds_path, check_installed=True,
purpose='creating a subdataset')
path_inrefds = path_under_rev_dataset(refds, path)
if path_inrefds is None:
yield dict(
res,
status='error',
message=(
"dataset containing given paths is not underneath "
"the reference dataset %s: %s",
dataset, text_type(path)),
)
return
# try to locate an immediate parent dataset
# we want to know this (irrespective of whether we plan on adding
# this new dataset to a parent) in order to avoid conflicts with
# a potentially absent/uninstalled subdataset of the parent
# in this location
# it will cost some filesystem traversal though...
parentds_path = rev_get_dataset_root(
op.normpath(op.join(text_type(path), os.pardir)))
if parentds_path:
prepo = GitRepo(parentds_path)
parentds_path = ut.Path(parentds_path)
# we cannot get away with a simple
# GitRepo.get_content_info(), as we need to detect
# uninstalled/added subdatasets too
check_path = ut.Path(path)
pstatus = prepo.status(
untracked='no',
# limit query to target path for a potentially massive speed-up
paths=[check_path.relative_to(parentds_path)])
if any(
check_path == p or check_path in p.parents
for p in pstatus):
# redo the check in a slower fashion, it is already broken
# let's take our time for a proper error message
conflict = [
p for p in pstatus
if check_path == p or check_path in p.parents]
res.update({
'status': 'error',
'message': (
'collision with content in parent dataset at %s: %s',
text_type(parentds_path),
[text_type(c) for c in conflict])})
yield res
return
# another set of check to see whether the target path is pointing
# into a known subdataset that is not around ATM
subds_status = {
parentds_path / k.relative_to(prepo.path)
for k, v in iteritems(pstatus)
if v.get('type', None) == 'dataset'}
check_paths = [check_path]
check_paths.extend(check_path.parents)
if any(p in subds_status for p in check_paths):
conflict = [p for p in check_paths if p in subds_status]
res.update({
'status': 'error',
'message': (
'collision with %s (dataset) in dataset %s',
text_type(conflict[0]),
text_type(parentds_path))})
yield res
return
# important to use the given Dataset object to avoid spurious ID
# changes with not-yet-materialized Datasets
tbds = dataset if isinstance(dataset, Dataset) and \
dataset.path == path else Dataset(text_type(path))
# don't create in non-empty directory without `force`:
if op.isdir(tbds.path) and listdir(tbds.path) != [] and not force:
res.update({
'status': 'error',
'message':
'will not create a dataset in a non-empty directory, use '
'`force` option to ignore'})
yield res
return
# stuff that we create and want to have tracked with git (not annex)
add_to_git = {}
if initopts is not None and isinstance(initopts, list):
initopts = {'_from_cmdline_': initopts}
# create and configure desired repository
if no_annex:
lgr.info("Creating a new git repo at %s", tbds.path)
tbrepo = GitRepo(
tbds.path,
url=None,
create=True,
create_sanity_checks=False,
git_opts=initopts,
fake_dates=fake_dates)
# place a .noannex file to indicate annex to leave this repo alone
stamp_path = ut.Path(tbrepo.path) / '.noannex'
stamp_path.touch()
add_to_git[stamp_path] = {
'type': 'file',
'state': 'untracked'}
else:
# always come with annex when created from scratch
lgr.info("Creating a new annex repo at %s", tbds.path)
tbrepo = AnnexRepo(
tbds.path,
url=None,
create=True,
create_sanity_checks=False,
# do not set backend here, to avoid a dedicated commit
backend=None,
# None causes version to be taken from config
version=None,
description=description,
git_opts=initopts,
fake_dates=fake_dates
)
# set the annex backend in .gitattributes as a staged change
tbrepo.set_default_backend(
cfg.obtain('datalad.repo.backend'),
persistent=True, commit=False)
add_to_git[tbds.repo.pathobj / '.gitattributes'] = {
'type': 'file',
'state': 'added'}
# make sure that v6 annex repos never commit content under .datalad
attrs_cfg = (
('config', 'annex.largefiles', 'nothing'),
('metadata/aggregate*', 'annex.largefiles', 'nothing'),
('metadata/objects/**', 'annex.largefiles',
'({})'.format(cfg.obtain(
'datalad.metadata.create-aggregate-annex-limit'))))
attrs = tbds.repo.get_gitattributes(
[op.join('.datalad', i[0]) for i in attrs_cfg])
set_attrs = []
for p, k, v in attrs_cfg:
if not attrs.get(
op.join('.datalad', p), {}).get(k, None) == v:
set_attrs.append((p, {k: v}))
if set_attrs:
tbds.repo.set_gitattributes(
set_attrs,
attrfile=op.join('.datalad', '.gitattributes'))
# prevent git annex from ever annexing .git* stuff (gh-1597)
attrs = tbds.repo.get_gitattributes('.git')
if not attrs.get('.git', {}).get(
'annex.largefiles', None) == 'nothing':
tbds.repo.set_gitattributes([
('**/.git*', {'annex.largefiles': 'nothing'})])
# must use the repo.pathobj as this will have resolved symlinks
add_to_git[tbds.repo.pathobj / '.gitattributes'] = {
'type': 'file',
'state': 'untracked'}
# record an ID for this repo for the afterlife
# to be able to track siblings and children
id_var = 'datalad.dataset.id'
# Note, that Dataset property `id` will change when we unset the
# respective config. Therefore store it before:
tbds_id = tbds.id
if id_var in tbds.config:
# make sure we reset this variable completely, in case of a
# re-create
tbds.config.unset(id_var, where='dataset')
if _seed is None:
# just the standard way
uuid_id = uuid.uuid1().urn.split(':')[-1]
else:
# Let's generate preseeded ones
uuid_id = str(uuid.UUID(int=random.getrandbits(128)))
tbds.config.add(
id_var,
tbds_id if tbds_id is not None else uuid_id,
where='dataset',
reload=False)
# make config overrides permanent in the repo config
# this is similar to what `annex init` does
# we are only doing this for config overrides and do not expose
# a dedicated argument, because it is sufficient for the cmdline
# and unnecessary for the Python API (there could simply be a
# subsequence ds.config.add() call)
for k, v in iteritems(tbds.config.overrides):
tbds.config.add(k, v, where='local', reload=False)
# all config manipulation is done -> fll reload
tbds.config.reload()
# must use the repo.pathobj as this will have resolved symlinks
add_to_git[tbds.repo.pathobj / '.datalad'] = {
'type': 'directory',
'state': 'untracked'}
# save everything, we need to do this now and cannot merge with the
# call below, because we may need to add this subdataset to a parent
# but cannot until we have a first commit
tbds.repo.save(
message='[DATALAD] new dataset',
git=True,
# we have to supply our own custom status, as the repo does
# not have a single commit yet and the is no HEAD reference
# TODO make `GitRepo.status()` robust to this state.
_status=add_to_git,
)
# the next only makes sense if we saved the created dataset,
# otherwise we have no committed state to be registered
# in the parent
if isinstance(dataset, Dataset) and dataset.path != tbds.path:
# we created a dataset in another dataset
# -> make submodule
for r in dataset.save(
path=tbds.path,
):
yield r
res.update({'status': 'ok'})
yield res
for cfg_proc_ in cfg_proc or []:
for r in tbds.run_procedure('cfg_' + cfg_proc_):
yield r