def diff_dataset(dataset, fr, to, constant_refs, path=None, annex=None, untracked='normal', recursive=False, recursion_limit=None, eval_file_type=True, reporting_order='depth-first'): """Internal helper to diff a dataset Parameters ---------- dataset : Dataset Dataset to perform the diff on. `fr` and `to` parameters are interpreted in the context of this dataset. fr : str Commit-ish to compare from. to : str Commit-ish to compare to. constant_refs : bool If True, `fr` and `to` will be passed on unmodified to diff operations on subdatasets. This can be useful with symbolic references like tags to report subdataset changes independent of superdataset changes. If False, `fr` and `to` will be translated to the subdataset commit-ish that match the given commit-ish in the superdataset. path : Path-like, optional Paths to constrain the diff to (see main diff() command). annex : str, optional Reporting mode for annex properties (see main diff() command). untracked : str, optional Reporting mode for untracked content (see main diff() command). recursive : bool, optional Flag to enable recursive operation (see main diff() command). recursion_limit : int, optional Recursion limit (see main diff() command). eval_file_type : bool, optional Whether to perform file type discrimination between real symlinks and symlinks representing annex'ed files. This can be expensive in datasets with many files. reporting_order : {'depth-first', 'breadth-first'}, optional By default, subdataset content records are reported after the record on the subdataset's submodule in a superdataset (depth-first). Alternatively, report all superdataset records first, before reporting any subdataset content records (breadth-first). Yields ------ dict DataLad result records. """ if reporting_order not in ('depth-first', 'breadth-first'): raise ValueError('Unknown reporting order: {}'.format(reporting_order)) ds = require_dataset(dataset, check_installed=True, purpose='difference reporting') # we cannot really perform any sorting of paths into subdatasets # or rejecting paths based on the state of the filesystem, as # we need to be able to compare with states that are not represented # in the worktree (anymore) if path: ps = [] # sort any path argument into the respective subdatasets for p in sorted(assure_list(path)): # it is important to capture the exact form of the # given path argument, before any normalization happens # distinguish rsync-link syntax to identify # a dataset as whole (e.g. 'ds') vs its # content (e.g. 'ds/') # special case is the root dataset, always report its content # changes orig_path = str(p) resolved_path = resolve_path(p, dataset) p = \ resolved_path, \ orig_path.endswith(op.sep) or resolved_path == ds.pathobj str_path = str(p[0]) root = get_dataset_root(str_path) if root is None: # no root, not possibly underneath the refds yield dict(action='status', path=str_path, refds=ds.path, status='error', message='path not underneath this dataset', logger=lgr) continue if path_under_rev_dataset(ds, str_path) is None: # nothing we support handling any further # there is only a single refds yield dict( path=str_path, refds=ds.path, action='diff', status='error', message=( "dataset containing given paths is not underneath " "the reference dataset %s: %s", ds, str_path), logger=lgr, ) continue ps.append(p) path = ps # TODO we might want to move away from the single-pass+immediate-yield # paradigm for this command. If we gather all information first, we # could do post-processing and detect when a file (same gitsha, or same # key) was copied/moved from another dataset. Another command (e.g. # save) could act on this information and also move/copy # availability information or at least enhance the respective commit # message with cross-dataset provenance info # cache to help avoid duplicate status queries content_info_cache = {} for res in _diff_ds( ds, fr, to, constant_refs, recursion_limit if recursion_limit is not None and recursive else -1 if recursive else 0, # TODO recode paths to repo path reference origpaths=None if not path else OrderedDict(path), untracked=untracked, annexinfo=annex, eval_file_type=eval_file_type, cache=content_info_cache, order=reporting_order): res.update( refds=ds.path, logger=lgr, action='diff', ) yield res
def __call__(path=None, initopts=None, *, force=False, description=None, dataset=None, annex=True, fake_dates=False, cfg_proc=None): # we only perform negative tests below no_annex = not annex if dataset: if isinstance(dataset, Dataset): ds = dataset else: ds = Dataset(dataset) refds_path = ds.path else: ds = refds_path = None # two major cases # 1. we got a `dataset` -> we either want to create it (path is None), # or another dataset in it (path is not None) # 2. we got no dataset -> we want to create a fresh dataset at the # desired location, either at `path` or PWD # sanity check first if no_annex: if description: raise ValueError("Incompatible arguments: cannot specify " "description for annex repo and declaring " "no annex repo.") if (isinstance(initopts, (list, tuple)) and '--bare' in initopts) or (isinstance(initopts, dict) and 'bare' in initopts): raise ValueError( "Creation of bare repositories is not supported. Consider " "one of the create-sibling commands, or use " "Git to init a bare repository and push an existing dataset " "into it.") if path: path = resolve_path(path, dataset) path = path if path \ else getpwd() if ds is None \ else refds_path # we know that we need to create a dataset at `path` assert (path is not None) # assure cfg_proc is a list (relevant if used via Python API) cfg_proc = ensure_list(cfg_proc) # prep for yield res = dict(action='create', path=str(path), logger=lgr, type='dataset', refds=refds_path) refds = None if refds_path and refds_path != str(path): refds = require_dataset(refds_path, check_installed=True, purpose='create a subdataset') path_inrefds = path_under_rev_dataset(refds, path) if path_inrefds is None: yield dict( res, status='error', message=( "dataset containing given paths is not underneath " "the reference dataset %s: %s", ds, str(path)), ) return # try to locate an immediate parent dataset # we want to know this (irrespective of whether we plan on adding # this new dataset to a parent) in order to avoid conflicts with # a potentially absent/uninstalled subdataset of the parent # in this location # it will cost some filesystem traversal though... parentds_path = get_dataset_root( op.normpath(op.join(str(path), os.pardir))) if parentds_path: prepo = GitRepo(parentds_path) parentds_path = Path(parentds_path) # we cannot get away with a simple # GitRepo.get_content_info(), as we need to detect # uninstalled/added subdatasets too check_path = Path(path) pstatus = prepo.status( untracked='no', # limit query to target path for a potentially massive speed-up paths=[check_path.relative_to(parentds_path)]) if (not pstatus.get(check_path, {}).get("type") == "dataset" and any(check_path == p or check_path in p.parents for p in pstatus)): # redo the check in a slower fashion, it is already broken # let's take our time for a proper error message conflict = [ p for p in pstatus if check_path == p or check_path in p.parents ] res.update({ 'status': 'error', 'message': ('collision with content in parent dataset at %s: %s', str(parentds_path), [str(c) for c in conflict]) }) yield res return if not force: # another set of check to see whether the target path is pointing # into a known subdataset that is not around ATM subds_status = { parentds_path / k.relative_to(prepo.path) for k, v in pstatus.items() if v.get('type', None) == 'dataset' } check_paths = [check_path] check_paths.extend(check_path.parents) if any(p in subds_status for p in check_paths): conflict = [p for p in check_paths if p in subds_status] res.update({ 'status': 'error', 'message': ('collision with %s (dataset) in dataset %s', str(conflict[0]), str(parentds_path)) }) yield res return # important to use the given Dataset object to avoid spurious ID # changes with not-yet-materialized Datasets tbds = ds if isinstance(ds, Dataset) and \ ds.path == path else Dataset(str(path)) # don't create in non-empty directory without `force`: if op.isdir(tbds.path) and listdir(tbds.path) != [] and not force: res.update({ 'status': 'error', 'message': 'will not create a dataset in a non-empty directory, use ' '`--force` option to ignore' }) yield res return # Check if specified cfg_proc(s) can be discovered, storing # the results so they can be used when the time comes to run # the procedure. If a procedure cannot be found, raise an # error to prevent creating the dataset. cfg_proc_specs = [] if cfg_proc: discovered_procs = tbds.run_procedure( discover=True, result_renderer='disabled', return_type='generator', ) for cfg_proc_ in cfg_proc: for discovered_proc in discovered_procs: if discovered_proc['procedure_name'] == 'cfg_' + cfg_proc_: cfg_proc_specs.append(discovered_proc) break else: raise ValueError("Cannot find procedure with name " "'%s'" % cfg_proc_) if initopts is not None and isinstance(initopts, list): initopts = {'_from_cmdline_': initopts} # Note for the code below: # OPT: be "smart" and avoid re-resolving .repo -- expensive in DataLad # Re-use tbrepo instance, do not use tbds.repo # create and configure desired repository # also provides initial set of content to be tracked with git (not annex) if no_annex: tbrepo, add_to_git = _setup_git_repo(path, initopts, fake_dates) else: tbrepo, add_to_git = _setup_annex_repo(path, initopts, fake_dates, description) # OPT: be "smart" and avoid re-resolving .repo -- expensive in DataLad # Note, must not happen earlier (before if) since "smart" it would not be tbds_config = tbds.config # record an ID for this repo for the afterlife # to be able to track siblings and children id_var = 'datalad.dataset.id' # Note, that Dataset property `id` will change when we unset the # respective config. Therefore store it before: tbds_id = tbds.id if id_var in tbds_config: # make sure we reset this variable completely, in case of a # re-create tbds_config.unset(id_var, scope='branch') if _seed is None: # just the standard way # use a fully random identifier (i.e. UUID version 4) uuid_id = str(uuid.uuid4()) else: # Let's generate preseeded ones uuid_id = str(uuid.UUID(int=random.getrandbits(128))) tbds_config.add(id_var, tbds_id if tbds_id is not None else uuid_id, scope='branch', reload=False) # make config overrides permanent in the repo config # this is similar to what `annex init` does # we are only doing this for config overrides and do not expose # a dedicated argument, because it is sufficient for the cmdline # and unnecessary for the Python API (there could simply be a # subsequence ds.config.add() call) for k, v in tbds_config.overrides.items(): tbds_config.add(k, v, scope='local', reload=False) # all config manipulation is done -> fll reload tbds_config.reload() # must use the repo.pathobj as this will have resolved symlinks add_to_git[tbrepo.pathobj / '.datalad'] = { 'type': 'directory', 'state': 'untracked' } # save everything, we need to do this now and cannot merge with the # call below, because we may need to add this subdataset to a parent # but cannot until we have a first commit tbrepo.save( message='[DATALAD] new dataset', git=True, # we have to supply our own custom status, as the repo does # not have a single commit yet and the is no HEAD reference # TODO make `GitRepo.status()` robust to this state. _status=add_to_git, ) for cfg_proc_spec in cfg_proc_specs: yield from tbds.run_procedure( cfg_proc_spec, result_renderer='disabled', return_type='generator', ) # the next only makes sense if we saved the created dataset, # otherwise we have no committed state to be registered # in the parent if isinstance(refds, Dataset) and refds.path != tbds.path: # we created a dataset in another dataset # -> make submodule yield from refds.save( path=tbds.path, return_type='generator', result_renderer='disabled', ) res.update({'status': 'ok'}) yield res
def __call__(path=None, initopts=None, force=False, description=None, dataset=None, no_annex=False, fake_dates=False, cfg_proc=None): refds_path = dataset.path if hasattr(dataset, 'path') else dataset # two major cases # 1. we got a `dataset` -> we either want to create it (path is None), # or another dataset in it (path is not None) # 2. we got no dataset -> we want to create a fresh dataset at the # desired location, either at `path` or PWD # sanity check first if no_annex: if description: raise ValueError("Incompatible arguments: cannot specify " "description for annex repo and declaring " "no annex repo.") if path: path = rev_resolve_path(path, dataset) path = path if path \ else getpwd() if dataset is None \ else refds_path # we know that we need to create a dataset at `path` assert (path is not None) # prep for yield res = dict(action='create', path=text_type(path), logger=lgr, type='dataset', refds=refds_path) refds = None if refds_path and refds_path != path: refds = require_dataset(refds_path, check_installed=True, purpose='creating a subdataset') path_inrefds = path_under_rev_dataset(refds, path) if path_inrefds is None: yield dict( res, status='error', message=( "dataset containing given paths is not underneath " "the reference dataset %s: %s", dataset, text_type(path)), ) return # try to locate an immediate parent dataset # we want to know this (irrespective of whether we plan on adding # this new dataset to a parent) in order to avoid conflicts with # a potentially absent/uninstalled subdataset of the parent # in this location # it will cost some filesystem traversal though... parentds_path = rev_get_dataset_root( op.normpath(op.join(text_type(path), os.pardir))) if parentds_path: prepo = GitRepo(parentds_path) parentds_path = ut.Path(parentds_path) # we cannot get away with a simple # GitRepo.get_content_info(), as we need to detect # uninstalled/added subdatasets too check_path = ut.Path(path) pstatus = prepo.status( untracked='no', # limit query to target path for a potentially massive speed-up paths=[check_path.relative_to(parentds_path)]) if any(check_path == p or check_path in p.parents for p in pstatus): # redo the check in a slower fashion, it is already broken # let's take our time for a proper error message conflict = [ p for p in pstatus if check_path == p or check_path in p.parents ] res.update({ 'status': 'error', 'message': ('collision with content in parent dataset at %s: %s', text_type(parentds_path), [text_type(c) for c in conflict]) }) yield res return # another set of check to see whether the target path is pointing # into a known subdataset that is not around ATM subds_status = { parentds_path / k.relative_to(prepo.path) for k, v in iteritems(pstatus) if v.get('type', None) == 'dataset' } check_paths = [check_path] check_paths.extend(check_path.parents) if any(p in subds_status for p in check_paths): conflict = [p for p in check_paths if p in subds_status] res.update({ 'status': 'error', 'message': ('collision with %s (dataset) in dataset %s', text_type(conflict[0]), text_type(parentds_path)) }) yield res return # important to use the given Dataset object to avoid spurious ID # changes with not-yet-materialized Datasets tbds = dataset if isinstance(dataset, Dataset) and \ dataset.path == path else Dataset(text_type(path)) # don't create in non-empty directory without `force`: if op.isdir(tbds.path) and listdir(tbds.path) != [] and not force: res.update({ 'status': 'error', 'message': 'will not create a dataset in a non-empty directory, use ' '`force` option to ignore' }) yield res return # stuff that we create and want to have tracked with git (not annex) add_to_git = {} if initopts is not None and isinstance(initopts, list): initopts = {'_from_cmdline_': initopts} # create and configure desired repository if no_annex: lgr.info("Creating a new git repo at %s", tbds.path) tbrepo = GitRepo(tbds.path, url=None, create=True, create_sanity_checks=False, git_opts=initopts, fake_dates=fake_dates) # place a .noannex file to indicate annex to leave this repo alone stamp_path = ut.Path(tbrepo.path) / '.noannex' stamp_path.touch() add_to_git[stamp_path] = {'type': 'file', 'state': 'untracked'} else: # always come with annex when created from scratch lgr.info("Creating a new annex repo at %s", tbds.path) tbrepo = AnnexRepo( tbds.path, url=None, create=True, create_sanity_checks=False, # do not set backend here, to avoid a dedicated commit backend=None, # None causes version to be taken from config version=None, description=description, git_opts=initopts, fake_dates=fake_dates) # set the annex backend in .gitattributes as a staged change tbrepo.set_default_backend(cfg.obtain('datalad.repo.backend'), persistent=True, commit=False) add_to_git[tbds.repo.pathobj / '.gitattributes'] = { 'type': 'file', 'state': 'added' } # make sure that v6 annex repos never commit content under .datalad attrs_cfg = (('config', 'annex.largefiles', 'nothing'), ( 'metadata/aggregate*', 'annex.largefiles', 'nothing' ), ('metadata/objects/**', 'annex.largefiles', '({})'.format( cfg.obtain('datalad.metadata.create-aggregate-annex-limit')))) attrs = tbds.repo.get_gitattributes( [op.join('.datalad', i[0]) for i in attrs_cfg]) set_attrs = [] for p, k, v in attrs_cfg: if not attrs.get(op.join('.datalad', p), {}).get(k, None) == v: set_attrs.append((p, {k: v})) if set_attrs: tbds.repo.set_gitattributes(set_attrs, attrfile=op.join( '.datalad', '.gitattributes')) # prevent git annex from ever annexing .git* stuff (gh-1597) attrs = tbds.repo.get_gitattributes('.git') if not attrs.get('.git', {}).get('annex.largefiles', None) == 'nothing': tbds.repo.set_gitattributes([('**/.git*', { 'annex.largefiles': 'nothing' })]) # must use the repo.pathobj as this will have resolved symlinks add_to_git[tbds.repo.pathobj / '.gitattributes'] = { 'type': 'file', 'state': 'untracked' } # record an ID for this repo for the afterlife # to be able to track siblings and children id_var = 'datalad.dataset.id' # Note, that Dataset property `id` will change when we unset the # respective config. Therefore store it before: tbds_id = tbds.id if id_var in tbds.config: # make sure we reset this variable completely, in case of a # re-create tbds.config.unset(id_var, where='dataset') if _seed is None: # just the standard way uuid_id = uuid.uuid1().urn.split(':')[-1] else: # Let's generate preseeded ones uuid_id = str(uuid.UUID(int=random.getrandbits(128))) tbds.config.add(id_var, tbds_id if tbds_id is not None else uuid_id, where='dataset', reload=False) # make config overrides permanent in the repo config # this is similar to what `annex init` does # we are only doing this for config overrides and do not expose # a dedicated argument, because it is sufficient for the cmdline # and unnecessary for the Python API (there could simply be a # subsequence ds.config.add() call) for k, v in iteritems(tbds.config.overrides): tbds.config.add(k, v, where='local', reload=False) # all config manipulation is done -> fll reload tbds.config.reload() # must use the repo.pathobj as this will have resolved symlinks add_to_git[tbds.repo.pathobj / '.datalad'] = { 'type': 'directory', 'state': 'untracked' } # save everything, we need to do this now and cannot merge with the # call below, because we may need to add this subdataset to a parent # but cannot until we have a first commit tbds.repo.save( message='[DATALAD] new dataset', git=True, # we have to supply our own custom status, as the repo does # not have a single commit yet and the is no HEAD reference # TODO make `GitRepo.status()` robust to this state. _status=add_to_git, ) # the next only makes sense if we saved the created dataset, # otherwise we have no committed state to be registered # in the parent if isinstance(refds, Dataset) and refds.path != tbds.path: # we created a dataset in another dataset # -> make submodule for r in refds.save(path=tbds.path, ): yield r res.update({'status': 'ok'}) yield res for cfg_proc_ in cfg_proc or []: for r in tbds.run_procedure('cfg_' + cfg_proc_): yield r
def __call__(path=None, dataset=None, annex=None, untracked='normal', recursive=False, recursion_limit=None, eval_subdataset_state='full'): # To the next white knight that comes in to re-implement `status` as a # special case of `diff`. There is one fundamental difference between # the two commands: `status` can always use the worktree as evident on # disk as a contraint (e.g. to figure out which subdataset a path is in) # `diff` cannot do that (everything need to be handled based on a # "virtual" representation of a dataset hierarchy). # MIH concludes that while `status` can be implemented as a special case # of `diff` doing so would complicate and slow down both `diff` and # `status`. So while the apparent almost code-duplication between the # two commands feels wrong, the benefit is speed. Any future RF should # come with evidence that speed does not suffer, and complexity stays # on a manageable level ds = require_dataset(dataset, check_installed=True, purpose='status reporting') paths_by_ds = OrderedDict() if path: # sort any path argument into the respective subdatasets for p in sorted(assure_list(path)): # it is important to capture the exact form of the # given path argument, before any normalization happens # for further decision logic below orig_path = text_type(p) p = rev_resolve_path(p, dataset) root = rev_get_dataset_root(text_type(p)) if root is None: # no root, not possibly underneath the refds yield dict(action='status', path=p, refds=ds.path, status='error', message='path not underneath this dataset', logger=lgr) continue else: if dataset and root == text_type(p) and \ not (orig_path.endswith(op.sep) or orig_path == "."): # the given path is pointing to a dataset # distinguish rsync-link syntax to identify # the dataset as whole (e.g. 'ds') vs its # content (e.g. 'ds/') super_root = rev_get_dataset_root(op.dirname(root)) if super_root: # the dataset identified by the path argument # is contained in a superdataset, and no # trailing path separator was found in the # argument -> user wants to address the dataset # as a whole (in the superdataset) root = super_root root = ut.Path(root) ps = paths_by_ds.get(root, []) ps.append(p) paths_by_ds[root] = ps else: paths_by_ds[ds.pathobj] = None queried = set() content_info_cache = {} while paths_by_ds: qdspath, qpaths = paths_by_ds.popitem(last=False) if qpaths and qdspath in qpaths: # this is supposed to be a full query, save some # cycles sifting through the actual path arguments qpaths = [] # try to recode the dataset path wrt to the reference # dataset # the path that it might have been located by could # have been a resolved path or another funky thing qds_inrefds = path_under_rev_dataset(ds, qdspath) if qds_inrefds is None: # nothing we support handling any further # there is only a single refds yield dict( path=text_type(qdspath), refds=ds.path, action='status', status='error', message=( "dataset containing given paths is not underneath " "the reference dataset %s: %s", ds, qpaths), logger=lgr, ) continue elif qds_inrefds != qdspath: # the path this dataset was located by is not how it would # be referenced underneath the refds (possibly resolved # realpath) -> recode all paths to be underneath the refds qpaths = [qds_inrefds / p.relative_to(qdspath) for p in qpaths] qdspath = qds_inrefds if qdspath in queried: # do not report on a single dataset twice continue qds = Dataset(text_type(qdspath)) for r in _yield_status( qds, qpaths, annex, untracked, recursion_limit if recursion_limit is not None else -1 if recursive else 0, queried, eval_subdataset_state, content_info_cache): yield dict( r, refds=ds.path, action='status', status='ok', )
def __call__(urls, *, dataset=None, path=None, overwrite=False, archive=False, save=True, message=None): from ..downloaders.http import HTTPDownloader from ..downloaders.providers import Providers ds = None if save or dataset: try: ds = require_dataset(dataset, check_installed=True, purpose='download urls') except NoDatasetFound: pass common_report = {"action": "download_url", "ds": ds} got_ds_instance = isinstance(dataset, Dataset) dir_is_target = not path or str(path).endswith(op.sep) path = str(resolve_path(path or op.curdir, ds=dataset)) if dir_is_target: # resolve_path() doesn't preserve trailing separators. Add one for # the download() call. path = path + op.sep urls = ensure_list_from_str(urls) if not dir_is_target: if len(urls) > 1: yield get_status_dict( status="error", message= ("When specifying multiple urls, --path should point to " "a directory target (with a trailing separator). Got %r", path), type="file", path=path, **common_report) return if archive: # make sure the file suffix indicated by a URL is preserved # so that any further archive processing doesn't have to # employ mime type inspection in order to determine the archive # type from datalad.support.network import URL suffixes = PurePosixPath(URL(urls[0]).path).suffixes if not Path(path).suffixes == suffixes: path += ''.join(suffixes) # we know that we have a single URL # download() would be fine getting an existing directory and # downloading the URL underneath it, but let's enforce a trailing # slash here for consistency. if op.isdir(path): yield get_status_dict( status="error", message=( "Non-directory path given (no trailing separator) " "but a directory with that name (after adding archive " "suffix) exists"), type="file", path=path, **common_report) return # TODO setup fancy ui.progressbars doing this in parallel and reporting overall progress # in % of urls which were already downloaded providers = Providers.from_config_files() downloaded_paths = [] path_urls = {} need_datalad_remote = False for url in urls: # somewhat "ugly" downloader = providers.get_provider(url).get_downloader(url) try: downloaded_path = downloader.download(url, path=path, overwrite=overwrite) except Exception as e: ce = CapturedException(e) yield get_status_dict(status="error", message=str(ce), type="file", path=path, exception=ce, **common_report) else: if not need_datalad_remote \ and (downloader.authenticator or downloader.credential or type(downloader) != HTTPDownloader): need_datalad_remote = True downloaded_paths.append(downloaded_path) path_urls[downloaded_path] = url yield get_status_dict(status="ok", type="file", path=downloaded_path, **common_report) if downloaded_paths and save and ds is not None: msg = message or """\ [DATALAD] Download URLs URLs: {}""".format("\n ".join(urls)) for r in Save()( downloaded_paths, message=msg, # ATTN: Pass the original dataset argument to # preserve relative path handling semantics. dataset=dataset, return_type="generator", result_renderer='disabled', result_xfm=None, result_filter=None, on_failure="ignore"): yield r ds_repo = ds.repo if isinstance(ds_repo, AnnexRepo): if need_datalad_remote: from datalad.customremotes.base import ( ensure_datalad_remote, ) ensure_datalad_remote(ds_repo, autoenable=True, encryption=None) if got_ds_instance: # Paths in `downloaded_paths` are already relative to the # dataset. rpaths = dict(zip(downloaded_paths, downloaded_paths)) else: # Paths in `downloaded_paths` are already relative to the # current working directory. Take these relative to the # dataset for use with the AnnexRepo method calls. rpaths = {} for orig_path, resolved in zip( downloaded_paths, resolve_path(downloaded_paths, ds=dataset)): rpath = path_under_rev_dataset(ds, resolved) if rpath: rpaths[str(rpath)] = orig_path else: lgr.warning("Path %s not under dataset %s", orig_path, ds) annex_paths = [ p for p, annexed in zip( rpaths, ds_repo.is_under_annex(list(rpaths.keys()))) if annexed ] if annex_paths: for path in annex_paths: url = path_urls[rpaths[path]] try: # The file is already present. This is just to # register the URL. ds_repo.add_url_to_file( path, url, # avoid batch mode for single files # https://github.com/datalad/datalad/issues/2849 batch=len(annex_paths) > 1, # bypass URL size check, we already have the file options=['--relaxed']) except CommandError as exc: lgr.warning("Registering %s with %s failed: %s", path, url, CapturedException(exc)) if archive: for path in annex_paths: yield from ds.add_archive_content( path, delete=True, on_failure='ignore', return_type='generator', result_renderer='disabled')
def get_paths_by_ds(refds, dataset_arg, paths, subdsroot_mode='rsync'): """Resolve and sort any paths into their containing datasets Any path will be associated (sorted into) its nearest containing dataset. It is irrelevant whether or not a path presently exists on the file system. However, only datasets that exist on the file system are used for sorting/association -- known, but non-existent subdatasets are not considered. Parameters ---------- refds: Dataset dataset_arg: Dataset or str or Path or None Any supported value given to a command's `dataset` argument. Given to `resolve_path()`. paths: list Any number of absolute or relative paths, in str-form or as Path instances, to be sorted into their respective datasets. See also the `subdsroot_mode` parameter. subdsroot_mode: {'rsync', 'super', 'sub'} Switch behavior for paths that are the root of a subdataset. By default ('rsync'), such a path is associated with its parent/superdataset, unless the path ends with a trailing directory separator, in which case it is sorted into the subdataset record (this resembles the path semantics of rsync, hence the label). In 'super' mode, the path is always placed with the superdataset record. Likewise, in 'sub' mode the path is always placed into the subdataset record. Returns ------- dict, list The first return value is the main result, a dictionary with root directories of all discovered datasets as keys and a list of the associated paths inside these datasets as values. Keys and values are normalized to be Path instances of absolute paths. The second return value is a list of all paths (again Path instances) that are not located underneath the reference dataset. """ ds_path = refds.path paths_by_ds = OrderedDict() errors = [] if not paths: # that was quick paths_by_ds[refds.pathobj] = None return paths_by_ds, errors # in order to guarantee proper path sorting, we first need to resolve all # of them (some may be str, some Path, some relative, some absolute) # step 1: normalize to unicode paths = map(ensure_unicode, paths) # step 2: resolve # for later comparison, we need to preserve the original value too paths = [(resolve_path(p, dataset_arg), str(p)) for p in paths] # OPT: store cache for dataset roots for each directory directly # listed in paths, or containing the path (if file) roots_cache = {} # sort any path argument into the respective subdatasets # sort by comparing the resolved Path instances, this puts top-level # paths first, leading to their datasets to be injected into the result # dict first for p, orig_path in sorted(paths, key=lambda x: x[0]): # TODO (left from implementing caching OPT): # Logic here sounds duplicated with discover_dataset_trace_to_targets # and even get_tree_roots of save. str_p = str(p) # query get_dataset_root caching for repeated queries within the same # directory if p.is_dir(): p_dir = str(p) else: # symlink, file, whatnot - seems to match logic in get_dataset_root p_dir = str(p.parent) try: root = roots_cache[p_dir] except KeyError: root = roots_cache[p_dir] = get_dataset_root(p_dir) # to become the root of the dataset that contains the path in question # in the context of (same basepath) as the reference dataset qds_inrefds = None if root is not None: qds_inrefds = path_under_rev_dataset(refds, root) if root is None or qds_inrefds is None: # no root, not possibly underneath the refds # or root that is not underneath/equal the reference dataset root errors.append(p) continue if root != qds_inrefds: # try to recode the dataset path wrt to the reference # dataset # the path that it might have been located by could # have been a resolved path or another funky thing # the path this dataset was located by is not how it would # be referenced underneath the refds (possibly resolved # realpath) -> recode all paths to be underneath the refds p = qds_inrefds / p.relative_to(root) root = qds_inrefds # Note: Compare to Dataset(root).path rather # than root to get same path normalization. if root == str_p and not Dataset(root).path == ds_path and ( subdsroot_mode == 'super' or (subdsroot_mode == 'rsync' and dataset_arg and not orig_path.endswith(op.sep))): # the given path is pointing to a subdataset # and we are either in 'super' mode, or in 'rsync' and found # rsync-link syntax to identify the dataset as whole # (e.g. 'ds') vs its content (e.g. 'ds/') root_dir = op.dirname(root) try: super_root = roots_cache[root_dir] except KeyError: super_root = roots_cache[root_dir] = get_dataset_root(root_dir) if super_root: # the dataset identified by the path argument # is contained in a superdataset, and no # trailing path separator was found in the # argument -> user wants to address the dataset # as a whole (in the superdataset) root = super_root root = ut.Path(root) ps = paths_by_ds.get(root, []) ps.append(p) paths_by_ds[root] = ps return paths_by_ds, errors
def __call__( path=None, dataset=None, annex=None, untracked='normal', recursive=False, recursion_limit=None, eval_subdataset_state='full'): # To the next white knight that comes in to re-implement `status` as a # special case of `diff`. There is one fundamental difference between # the two commands: `status` can always use the worktree as evident on # disk as a contraint (e.g. to figure out which subdataset a path is in) # `diff` cannot do that (everything need to be handled based on a # "virtual" representation of a dataset hierarchy). # MIH concludes that while `status` can be implemented as a special case # of `diff` doing so would complicate and slow down both `diff` and # `status`. So while the apparent almost code-duplication between the # two commands feels wrong, the benefit is speed. Any future RF should # come with evidence that speed does not suffer, and complexity stays # on a manageable level ds = require_dataset( dataset, check_installed=True, purpose='status reporting') paths_by_ds = OrderedDict() if path: # sort any path argument into the respective subdatasets for p in sorted(assure_list(path)): # it is important to capture the exact form of the # given path argument, before any normalization happens # for further decision logic below orig_path = text_type(p) p = rev_resolve_path(p, dataset) root = rev_get_dataset_root(text_type(p)) if root is None: # no root, not possibly underneath the refds yield dict( action='status', path=p, refds=ds.path, status='error', message='path not underneath this dataset', logger=lgr) continue else: if dataset and root == text_type(p) and \ not (orig_path.endswith(op.sep) or orig_path == "."): # the given path is pointing to a dataset # distinguish rsync-link syntax to identify # the dataset as whole (e.g. 'ds') vs its # content (e.g. 'ds/') super_root = rev_get_dataset_root(op.dirname(root)) if super_root: # the dataset identified by the path argument # is contained in a superdataset, and no # trailing path separator was found in the # argument -> user wants to address the dataset # as a whole (in the superdataset) root = super_root root = ut.Path(root) ps = paths_by_ds.get(root, []) ps.append(p) paths_by_ds[root] = ps else: paths_by_ds[ds.pathobj] = None queried = set() content_info_cache = {} while paths_by_ds: qdspath, qpaths = paths_by_ds.popitem(last=False) if qpaths and qdspath in qpaths: # this is supposed to be a full query, save some # cycles sifting through the actual path arguments qpaths = [] # try to recode the dataset path wrt to the reference # dataset # the path that it might have been located by could # have been a resolved path or another funky thing qds_inrefds = path_under_rev_dataset(ds, qdspath) if qds_inrefds is None: # nothing we support handling any further # there is only a single refds yield dict( path=text_type(qdspath), refds=ds.path, action='status', status='error', message=( "dataset containing given paths is not underneath " "the reference dataset %s: %s", ds, qpaths), logger=lgr, ) continue elif qds_inrefds != qdspath: # the path this dataset was located by is not how it would # be referenced underneath the refds (possibly resolved # realpath) -> recode all paths to be underneath the refds qpaths = [qds_inrefds / p.relative_to(qdspath) for p in qpaths] qdspath = qds_inrefds if qdspath in queried: # do not report on a single dataset twice continue qds = Dataset(text_type(qdspath)) for r in _yield_status( qds, qpaths, annex, untracked, recursion_limit if recursion_limit is not None else -1 if recursive else 0, queried, eval_subdataset_state, content_info_cache): yield dict( r, refds=ds.path, action='status', status='ok', )
def __call__( path=None, initopts=None, force=False, description=None, dataset=None, no_annex=False, fake_dates=False, cfg_proc=None ): refds_path = dataset.path if hasattr(dataset, 'path') else dataset # two major cases # 1. we got a `dataset` -> we either want to create it (path is None), # or another dataset in it (path is not None) # 2. we got no dataset -> we want to create a fresh dataset at the # desired location, either at `path` or PWD # sanity check first if no_annex: if description: raise ValueError("Incompatible arguments: cannot specify " "description for annex repo and declaring " "no annex repo.") if path: path = rev_resolve_path(path, dataset) path = path if path \ else getpwd() if dataset is None \ else refds_path # we know that we need to create a dataset at `path` assert(path is not None) # prep for yield res = dict(action='create', path=text_type(path), logger=lgr, type='dataset', refds=refds_path) refds = None if refds_path and refds_path != path: refds = require_dataset( refds_path, check_installed=True, purpose='creating a subdataset') path_inrefds = path_under_rev_dataset(refds, path) if path_inrefds is None: yield dict( res, status='error', message=( "dataset containing given paths is not underneath " "the reference dataset %s: %s", dataset, text_type(path)), ) return # try to locate an immediate parent dataset # we want to know this (irrespective of whether we plan on adding # this new dataset to a parent) in order to avoid conflicts with # a potentially absent/uninstalled subdataset of the parent # in this location # it will cost some filesystem traversal though... parentds_path = rev_get_dataset_root( op.normpath(op.join(text_type(path), os.pardir))) if parentds_path: prepo = GitRepo(parentds_path) parentds_path = ut.Path(parentds_path) # we cannot get away with a simple # GitRepo.get_content_info(), as we need to detect # uninstalled/added subdatasets too check_path = ut.Path(path) pstatus = prepo.status( untracked='no', # limit query to target path for a potentially massive speed-up paths=[check_path.relative_to(parentds_path)]) if any( check_path == p or check_path in p.parents for p in pstatus): # redo the check in a slower fashion, it is already broken # let's take our time for a proper error message conflict = [ p for p in pstatus if check_path == p or check_path in p.parents] res.update({ 'status': 'error', 'message': ( 'collision with content in parent dataset at %s: %s', text_type(parentds_path), [text_type(c) for c in conflict])}) yield res return # another set of check to see whether the target path is pointing # into a known subdataset that is not around ATM subds_status = { parentds_path / k.relative_to(prepo.path) for k, v in iteritems(pstatus) if v.get('type', None) == 'dataset'} check_paths = [check_path] check_paths.extend(check_path.parents) if any(p in subds_status for p in check_paths): conflict = [p for p in check_paths if p in subds_status] res.update({ 'status': 'error', 'message': ( 'collision with %s (dataset) in dataset %s', text_type(conflict[0]), text_type(parentds_path))}) yield res return # important to use the given Dataset object to avoid spurious ID # changes with not-yet-materialized Datasets tbds = dataset if isinstance(dataset, Dataset) and \ dataset.path == path else Dataset(text_type(path)) # don't create in non-empty directory without `force`: if op.isdir(tbds.path) and listdir(tbds.path) != [] and not force: res.update({ 'status': 'error', 'message': 'will not create a dataset in a non-empty directory, use ' '`force` option to ignore'}) yield res return # stuff that we create and want to have tracked with git (not annex) add_to_git = {} if initopts is not None and isinstance(initopts, list): initopts = {'_from_cmdline_': initopts} # create and configure desired repository if no_annex: lgr.info("Creating a new git repo at %s", tbds.path) tbrepo = GitRepo( tbds.path, url=None, create=True, create_sanity_checks=False, git_opts=initopts, fake_dates=fake_dates) # place a .noannex file to indicate annex to leave this repo alone stamp_path = ut.Path(tbrepo.path) / '.noannex' stamp_path.touch() add_to_git[stamp_path] = { 'type': 'file', 'state': 'untracked'} else: # always come with annex when created from scratch lgr.info("Creating a new annex repo at %s", tbds.path) tbrepo = AnnexRepo( tbds.path, url=None, create=True, create_sanity_checks=False, # do not set backend here, to avoid a dedicated commit backend=None, # None causes version to be taken from config version=None, description=description, git_opts=initopts, fake_dates=fake_dates ) # set the annex backend in .gitattributes as a staged change tbrepo.set_default_backend( cfg.obtain('datalad.repo.backend'), persistent=True, commit=False) add_to_git[tbds.repo.pathobj / '.gitattributes'] = { 'type': 'file', 'state': 'added'} # make sure that v6 annex repos never commit content under .datalad attrs_cfg = ( ('config', 'annex.largefiles', 'nothing'), ('metadata/aggregate*', 'annex.largefiles', 'nothing'), ('metadata/objects/**', 'annex.largefiles', '({})'.format(cfg.obtain( 'datalad.metadata.create-aggregate-annex-limit')))) attrs = tbds.repo.get_gitattributes( [op.join('.datalad', i[0]) for i in attrs_cfg]) set_attrs = [] for p, k, v in attrs_cfg: if not attrs.get( op.join('.datalad', p), {}).get(k, None) == v: set_attrs.append((p, {k: v})) if set_attrs: tbds.repo.set_gitattributes( set_attrs, attrfile=op.join('.datalad', '.gitattributes')) # prevent git annex from ever annexing .git* stuff (gh-1597) attrs = tbds.repo.get_gitattributes('.git') if not attrs.get('.git', {}).get( 'annex.largefiles', None) == 'nothing': tbds.repo.set_gitattributes([ ('**/.git*', {'annex.largefiles': 'nothing'})]) # must use the repo.pathobj as this will have resolved symlinks add_to_git[tbds.repo.pathobj / '.gitattributes'] = { 'type': 'file', 'state': 'untracked'} # record an ID for this repo for the afterlife # to be able to track siblings and children id_var = 'datalad.dataset.id' # Note, that Dataset property `id` will change when we unset the # respective config. Therefore store it before: tbds_id = tbds.id if id_var in tbds.config: # make sure we reset this variable completely, in case of a # re-create tbds.config.unset(id_var, where='dataset') if _seed is None: # just the standard way uuid_id = uuid.uuid1().urn.split(':')[-1] else: # Let's generate preseeded ones uuid_id = str(uuid.UUID(int=random.getrandbits(128))) tbds.config.add( id_var, tbds_id if tbds_id is not None else uuid_id, where='dataset', reload=False) # make config overrides permanent in the repo config # this is similar to what `annex init` does # we are only doing this for config overrides and do not expose # a dedicated argument, because it is sufficient for the cmdline # and unnecessary for the Python API (there could simply be a # subsequence ds.config.add() call) for k, v in iteritems(tbds.config.overrides): tbds.config.add(k, v, where='local', reload=False) # all config manipulation is done -> fll reload tbds.config.reload() # must use the repo.pathobj as this will have resolved symlinks add_to_git[tbds.repo.pathobj / '.datalad'] = { 'type': 'directory', 'state': 'untracked'} # save everything, we need to do this now and cannot merge with the # call below, because we may need to add this subdataset to a parent # but cannot until we have a first commit tbds.repo.save( message='[DATALAD] new dataset', git=True, # we have to supply our own custom status, as the repo does # not have a single commit yet and the is no HEAD reference # TODO make `GitRepo.status()` robust to this state. _status=add_to_git, ) # the next only makes sense if we saved the created dataset, # otherwise we have no committed state to be registered # in the parent if isinstance(dataset, Dataset) and dataset.path != tbds.path: # we created a dataset in another dataset # -> make submodule for r in dataset.save( path=tbds.path, ): yield r res.update({'status': 'ok'}) yield res for cfg_proc_ in cfg_proc or []: for r in tbds.run_procedure('cfg_' + cfg_proc_): yield r