class CreateTestDataset(Interface): """Create test (meta-)dataset. """ # XXX prevent common args from being added to the docstring _no_eval_results = True _params_ = dict( path=Parameter( args=("path",), doc="path/name where to create (if specified, must not exist)", constraints=EnsureStr() | EnsureNone()), spec=Parameter( args=("--spec",), doc="""\ spec for hierarchy, defined as a min-max (min could be omitted to assume 0) defining how many (random number from min to max) of sub-datasets to generate at any given level of the hierarchy. Each level separated from each other with /. Example: 1-3/-2 would generate from 1 to 3 subdatasets at the top level, and up to two within those at the 2nd level """, constraints=EnsureStr() | EnsureNone()), seed=Parameter( args=("--seed",), doc="""seed for rng""", constraints=EnsureInt() | EnsureNone()), ) @staticmethod def __call__(path=None, spec=None, seed=None): levels = _parse_spec(spec) if seed is not None: # TODO: if to be used within a bigger project we shouldn't seed main RNG random.seed(seed) if path is None: kw = get_tempfile_kwargs({}, prefix="ds") path = tempfile.mkdtemp(**kw) else: # so we don't override anything assert not exists(path) os.makedirs(path) # now we should just make it happen and return list of all the datasets return list(_makeds(path, levels)) @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui if res is None: res = [] if not len(res): ui.message("No repos were created... oops") return items = '\n'.join(map(str, res)) msg = "{n} installed {obj} available at\n{items}".format( obj='items are' if len(res) > 1 else 'item is', n=len(res), items=items) ui.message(msg)
args=("-D", "--description",), constraints=EnsureStr() | EnsureNone(), doc="""short description to use for a dataset location. Its primary purpose is to help humans to identify a dataset copy (e.g., "mike's dataset on lab server"). Note that when a dataset is published, this information becomes available on the remote side.""") recursion_flag = Parameter( args=("-r", "--recursive",), action="store_true", doc="""if set, recurse into potential subdataset""") recursion_limit = Parameter( args=("--recursion-limit",), metavar="LEVELS", constraints=EnsureInt() | EnsureNone(), doc="""limit recursion into subdataset to the given number of levels""") shared_access_opt = Parameter( args=('--shared-access',), metavar='MODE', doc="""configure shared access to a dataset, see `git init --shared` documentation for complete details on the supported scenarios. Possible values include: 'false', 'true', 'group', and 'all'""") super_datasets_flag = Parameter( args=("-S", "--super-datasets",), action="store_true", doc="""if set, save a change in a dataset also in its superdataset""") git_opts = Parameter(
class Search(Interface): """Search dataset metadata DataLad can search metadata extracted from a dataset and/or aggregated into a superdataset (see the `aggregate-metadata` command). This makes it possible to discover datasets, or individual files in a dataset even when they are not available locally. Ultimately DataLad metadata are a graph of linked data structures. However, this command does not (yet) support queries that can exploit all information stored in the metadata. At the moment the following search modes are implemented that represent different trade-offs between the expressiveness of a query and the computational and storage resources required to execute a query. - egrep (default) - egrepcs [case-sensitive egrep] - textblob - autofield An alternative default mode can be configured by tuning the configuration variable 'datalad.search.default-mode':: [datalad "search"] default-mode = egrepcs Each search mode has its own default configuration for what kind of documents to query. The respective default can be changed via configuration variables:: [datalad "search"] index-<mode_name>-documenttype = (all|datasets|files) *Mode: egrep/egrepcs* These search modes are largely ignorant of the metadata structure, and simply perform matching of a search pattern against a flat string-representation of metadata. This is advantageous when the query is simple and the metadata structure is irrelevant, or precisely known. Moreover, it does not require a search index, hence results can be reported without an initial latency for building a search index when the underlying metadata has changed (e.g. due to a dataset update). By default, these search modes only consider datasets and do not investigate records for individual files for speed reasons. Search results are reported in the order in which they were discovered. Queries can make use of Python regular expression syntax (https://docs.python.org/3/library/re.html). In `egrep` mode, matching is case-insensitive when the query does not contain upper case characters, but is case-sensitive when it does. In `egrepcs` mode, matching is always case-sensitive. Expressions will match anywhere in a metadata string, not only at the start. When multiple queries are given, all queries have to match for a search hit (AND behavior). It is possible to search individual metadata key/value items by prefixing the query with a metadata key name, separated by a colon (':'). The key name can also be a regular expression to match multiple keys. A query match happens when any value of an item with a matching key name matches the query (OR behavior). See examples for more information. Examples: Query for (what happens to be) an author:: % datalad search haxby Queries are case-INsensitive when the query contains no upper case characters, and can be regular expressions. Use `egrepcs` mode when it is desired to perform a case-sensitive lowercase match:: % datalad search --mode egrepcs halchenko.*haxby This search mode performs NO analysis of the metadata content. Therefore queries can easily fail to match. For example, the above query implicitly assumes that authors are listed in alphabetical order. If that is the case (which may or may not be true), the following query would yield NO hits:: % datalad search Haxby.*Halchenko The ``textblob`` search mode represents an alternative that is more robust in such cases. For more complex queries multiple query expressions can be provided that all have to match to be considered a hit (AND behavior). This query discovers all files (non-default behavior) that match 'bids.type=T1w' AND 'nifti1.qform_code=scanner':: % datalad -c datalad.search.index-egrep-documenttype=all search bids.type:T1w nifti1.qform_code:scanner Key name selectors can also be expressions, which can be used to select multiple keys or construct "fuzzy" queries. In such cases a query matches when any item with a matching key matches the query (OR behavior). However, multiple queries are always evaluated using an AND conjunction. The following query extends the example above to match any files that have either 'nifti1.qform_code=scanner' or 'nifti1.sform_code=scanner':: % datalad -c datalad.search.index-egrep-documenttype=all search bids.type:T1w nifti1.(q|s)form_code:scanner *Mode: textblob* This search mode is very similar to the ``egrep`` mode, but with a few key differences. A search index is built from the string-representation of metadata records. By default, only datasets are included in this index, hence the indexing is usually completed within a few seconds, even for hundreds of datasets. This mode uses its own query language (not regular expressions) that is similar to other search engines. It supports logical conjunctions and fuzzy search terms. More information on this is available from the Whoosh project (search engine implementation): - Description of the Whoosh query language: http://whoosh.readthedocs.io/en/latest/querylang.html) - Description of a number of query language customizations that are enabled in DataLad, such as, fuzzy term matching: http://whoosh.readthedocs.io/en/latest/parsing.html#common-customizations Importantly, search hits are scored and reported in order of descending relevance, hence limiting the number of search results is more meaningful than in the 'egrep' mode and can also reduce the query duration. Examples: Search for (what happens to be) two authors, regardless of the order in which those names appear in the metadata:: % datalad search --mode textblob halchenko haxby Fuzzy search when you only have an approximate idea what you are looking for or how it is spelled:: % datalad search --mode textblob haxbi~ Very fuzzy search, when you are basically only confident about the first two characters and how it sounds approximately (or more precisely: allow for three edits and require matching of the first two characters):: % datalad search --mode textblob haksbi~3/2 Combine fuzzy search with logical constructs:: % datalad search --mode textblob 'haxbi~ AND (hanke OR halchenko)' *Mode: autofield* This mode is similar to the 'textblob' mode, but builds a vastly more detailed search index that represents individual metadata variables as individual fields. By default, this search index includes records for datasets and individual fields, hence it can grow very quickly into a huge structure that can easily take an hour or more to build and require more than a GB of storage. However, limiting it to documents on datasets (see above) retains the enhanced expressiveness of queries while dramatically reducing the resource demands. Examples: List names of search index fields (auto-discovered from the set of indexed datasets):: % datalad search --mode autofield --show-keys name Fuzzy search for datasets with an author that is specified in a particular metadata field:: % datalad search --mode autofield bids.author:haxbi~ type:dataset Search for individual files that carry a particular description prefix in their 'nifti1' metadata:: % datalad search --mode autofield nifti1.description:FSL* type:file *Reporting* Search hits are returned as standard DataLad results. On the command line the '--output-format' (or '-f') option can be used to tweak results for further processing. Examples: Format search hits as a JSON stream (one hit per line):: % datalad -f json search haxby Custom formatting: which terms matched the query of particular results. Useful for investigating fuzzy search results:: $ datalad -f '{path}: {query_matched}' search --mode autofield bids.author:haxbi~ """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to perform the query operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), query=Parameter( args=("query", ), metavar='QUERY', nargs="*", doc="""query string, supported syntax and features depends on the selected search mode (see documentation)"""), force_reindex=Parameter( args=("--reindex", ), dest='force_reindex', action='store_true', doc="""force rebuilding the search index, even if no change in the dataset's state has been detected, for example, when the index documenttype configuration has changed."""), max_nresults=Parameter( args=("--max-nresults", ), doc="""maxmimum number of search results to report. Setting this to 0 will report all search matches. Depending on the mode this can search substantially slower. If not specified, a mode-specific default setting will be used.""", constraints=EnsureInt() | EnsureNone()), mode=Parameter( args=("--mode", ), choices=('egrep', 'textblob', 'autofield'), doc="""Mode of search index structure and content. See section SEARCH MODES for details. """), full_record=Parameter( args=("--full-record", '-f'), action='store_true', doc="""If set, return the full metadata record for each search hit. Depending on the search mode this might require additional queries. By default, only data that is available to the respective search modes is returned. This always includes essential information, such as the path and the type."""), show_keys=Parameter( args=('--show-keys', ), choices=('name', 'short', 'full'), default=None, doc="""if given, a list of known search keys is shown. If 'name' - only the name is printed one per line. If 'short' or 'full', statistics (in how many datasets, and how many unique values) are printed. 'short' truncates the listing of unique values. No other action is performed (except for reindexing), even if other arguments are given. Each key is accompanied by a term definition in parenthesis (TODO). In most cases a definition is given in the form of a URL. If an ontology definition for a term is known, this URL can resolve to a webpage that provides a comprehensive definition of the term. However, for speed reasons term resolution is solely done on information contained in a local dataset's metadata, and definition URLs might be outdated or point to no longer existing resources.""" ), show_query=Parameter( args=('--show-query', ), action='store_true', doc="""if given, the formal query that was generated from the given query string is shown, but not actually executed. This is mostly useful for debugging purposes."""), ) @staticmethod @datasetmethod(name='search') @eval_results def __call__(query=None, dataset=None, force_reindex=False, max_nresults=None, mode=None, full_record=False, show_keys=None, show_query=False): try: ds = require_dataset(dataset, check_installed=True, purpose='dataset search') if ds.id is None: raise NoDatasetArgumentFound( "This does not seem to be a dataset (no DataLad dataset ID " "found). 'datalad create --force %s' can initialize " "this repository as a DataLad dataset" % ds.path) except NoDatasetArgumentFound: for r in _search_from_virgin_install(dataset, query): yield r return if mode is None: # let's get inspired by what the dataset/user think is # default mode = ds.config.obtain('datalad.search.default-mode') if mode == 'egrep': searcher = _EGrepSearch elif mode == 'egrepcs': searcher = _EGrepCSSearch elif mode == 'textblob': searcher = _BlobSearch elif mode == 'autofield': searcher = _AutofieldSearch else: raise ValueError('unknown search mode "{}"'.format(mode)) searcher = searcher(ds, force_reindex=force_reindex) if show_keys: searcher.show_keys(show_keys) return if not query: return if show_query: print(repr(searcher.get_query(query))) return for r in searcher(query, max_nresults=max_nresults, full_record=full_record): yield r
class Get(Interface): """Get any dataset content (files/directories/subdatasets). This command only operates on dataset content. To obtain a new independent dataset from some source use the `install` command. By default this command operates recursively within a dataset, but not across potential subdatasets, i.e. if a directory is provided, all files in the directory are obtained. Recursion into subdatasets is supported too. If enabled, relevant subdatasets are detected and installed in order to fulfill a request. Known data locations for each requested file are evaluated and data are obtained from some available location (according to git-annex configuration and possibly assigned remote priorities), unless a specific source is specified. .. note:: Power-user info: This command uses :command:`git annex get` to fulfill file handles. """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar="PATH", doc="""specify the dataset to perform the add operation on, in which case `path` arguments are interpreted as being relative to this dataset. If no dataset is given, an attempt is made to identify a dataset for each input `path`""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path", ), metavar="PATH", doc="""path/name of the requested dataset component. The component must already be known to a dataset. To add new components to a dataset use the `add` command""", nargs="*", constraints=EnsureStr() | EnsureNone()), source=Parameter( args=( "-s", "--source", ), metavar="LABEL", doc="""label of the data source to be used to fulfill requests. This can be the name of a dataset :term:`sibling` or another known source""", constraints=EnsureStr() | EnsureNone()), recursive=recursion_flag, recursion_limit=Parameter( args=("--recursion-limit", ), metavar="LEVELS", constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(), doc="""limit recursion into subdataset to the given number of levels. Alternatively, 'existing' will limit recursion to subdatasets that already existed on the filesystem at the start of processing, and prevent new subdatasets from being obtained recursively."""), get_data=Parameter( args=( "-n", "--no-data", ), dest='get_data', action='store_false', doc= """whether to obtain data for all file handles. If disabled, `get` operations are limited to dataset handles.[CMD: This option prevents data for file handles from being obtained CMD]"""), description=location_description, reckless=reckless_opt, # git_opts=git_opts, # annex_opts=annex_opts, # annex_get_opts=annex_get_opts, jobs=jobs_opt, verbose=verbose) # Note: May be use 'git annex find --not --in here' to have a list of all # files to actually get and give kind of a progress in terms of number # files processed ... @staticmethod @datasetmethod(name='get') @eval_results def __call__( path=None, source=None, dataset=None, recursive=False, recursion_limit=None, get_data=True, description=None, reckless=False, #git_opts=None, #annex_opts=None, #annex_get_opts=None, jobs='auto', verbose=False, ): # IMPLEMENTATION CONCEPT: # # 1. Sort the world into existing handles and the rest # 2. Try locate missing handles (obtain subdatasets along the way) # 3. Expand into subdatasets with recursion enables (potentially # obtain even more subdatasets # 4. Shoot info of which handles to get in each subdataset to, # git-annex, once at the very end refds_path = Interface.get_refds_path(dataset) if not (dataset or path): raise InsufficientArgumentsError( "Neither dataset nor target path(s) provided") if dataset and not path: # act on the whole dataset if nothing else was specified path = refds_path # remember which results we already reported, to avoid duplicates yielded_ds = [] to_get = [] unavailable_paths = [] for ap in AnnotatePaths.__call__( path=path, dataset=refds_path, recursive=recursive, recursion_limit=recursion_limit, action='get', # NOTE: Do not act upon unavailable paths yet! Done below after # testing which ones could be obtained unavailable_path_status='', nondataset_path_status='impossible', return_type='generator', on_failure='ignore'): if ap.get('status', None): # we know what to report already yield ap continue if ap.get('state', None) == 'absent' and ap.get( 'raw_input', False): # if this wasn't found, but directly requested, queue for further # exploration unavailable_paths.append(ap) continue if ap.get('type', None) == 'dataset' and \ GitRepo.is_valid_repo(ap['path']) and \ not ap['path'] == refds_path: # do not report what hasn't arived yet # also do not report the base dataset that is already # present -- no surprise yield dict(ap, status='notneeded', logger=lgr, message='already installed') yielded_ds.append(ap['path']) ap['process_content'] = get_data to_get.append(ap) # explore the unknown for ap in sorted(unavailable_paths, key=lambda x: x['path']): lgr.debug("Investigate yet unavailable path %s", ap) # how close can we get? dspath = ap.get('parentds', get_dataset_root(ap['path'])) if dspath is None: # nothing we can do for this path continue lgr.debug("Found containing dataset %s for path %s", dspath, ap['path']) ds = Dataset(dspath) # now actually obtain whatever is necessary to get to this path containing_ds = [dspath] for res in _install_necessary_subdatasets(ds, ap['path'], reckless, refds_path, description=description): # yield immediately so errors could be acted upon outside, before # we continue if not (res['type'] == 'dataset' and res['path'] in yielded_ds): # unless we reported on this dataset before if res['type'] == 'dataset': # make a record, recursive below might now want to report # a 'notneeded' yielded_ds.append(res['path']) yield res # update to the current innermost dataset containing_ds.append(res['path']) if len(containing_ds) < 2: # no subdataset was installed, hence if the path was unavailable # before it still is, no need to bother git annex ap.update(status='impossible', message='path does not exist') yield ap continue # important to only do the next for the innermost subdataset # as the `recursive` logic below relies on that! # set the correct parent, for a dataset this would be the second-last # reported subdataset ap.update(parentds=containing_ds[-1]) if containing_ds[-1] == ap['path']: # the path actually refers to the last installed dataset ap.update(parentds=containing_ds[-2], process_content=get_data, type='dataset') to_get.append(ap) # results of recursive installation of yet undiscovered datasets rec_get = [] if recursive and not recursion_limit == 'existing': # obtain any subdatasets underneath the paths given inside the # subdatasets that we know already exist # unless we do not want recursion into not-yet-installed datasets for ap in sorted(to_get, key=lambda x: x['path']): if ap['type'] not in ('dataset', 'directory') or not ap.get( 'raw_input', False): # a non-directory cannot have content underneath # also we do NOT want to recurse into anything that was specifically # requested, to avoid duplication continue subds = Dataset(ap['path'] if ap['type'] == 'dataset' else ap['parentds']) lgr.info("Installing %s%s recursively", subds, (" underneath %s" % ap['path'] if subds.path != ap['path'] else "")) for res in _recursive_install_subds_underneath( subds, # `ap['path']` was explicitly given as input # we count recursions from the input, hence we # can start with the full number recursion_limit, reckless, start=ap['path'], refds_path=refds_path, description=description): # yield immediately so errors could be acted upon # outside, before we continue if not (res['type'] == 'dataset' and res['path'] in yielded_ds): # unless we reported on this dataset before if res['type'] == 'dataset': # make a record yielded_ds.append(res['path']) yield res if not (res['status'] == 'ok' and res['type'] == 'dataset'): # not a dataset that was just installed, we just reported it # upstairs, and can ignore it from now on continue # paranoia, so popular these days... assert GitRepo.is_valid_repo(res['path']) # keep a copy of the install record for `get` later on get_ap = { k: v for k, v in res.items() if not k == 'status' } get_ap['process_content'] = get_data rec_get.append(get_ap) if not get_data: # done already return # merge the two AP lists to_get.extend(rec_get) # sort into datasets content_by_ds, ds_props, completed, nondataset_paths = \ annotated2content_by_ds( to_get, refds_path=refds_path) assert (not completed) # hand over to git-annex, get files content, # report files in git as 'notneeded' to get for ds_path in sorted(content_by_ds.keys()): ds = Dataset(ds_path) # grab content, ignore subdataset entries content = [ ap['path'] for ap in content_by_ds[ds_path] if ap.get('type', None) != 'dataset' or ap['path'] == ds.path ] if not content: # cut this short should there be nothing continue # needs to be an annex to get content if not isinstance(ds.repo, AnnexRepo): for r in results_from_paths( content, status='notneeded', message="no dataset annex, content already present", action='get', logger=lgr, refds=refds_path): yield r continue respath_by_status = {} for res in ds.repo.get(content, options=['--from=%s' % source] if source else [], jobs=jobs): res = annexjson2result(res, ds, type='file', logger=lgr, refds=refds_path) success = success_status_map[res['status']] # TODO: in case of some failed commands (e.g. get) there might # be no path in the record. yoh has only vague idea of logic # here so just checks for having 'path', but according to # results_from_annex_noinfo, then it would be assumed that # `content` was acquired successfully, which is not the case if 'path' in res: respath_by_status[success] = \ respath_by_status.get(success, []) + [res['path']] yield res for r in results_from_annex_noinfo( ds, content, respath_by_status, dir_fail_msg='could not get some content in %s %s', noinfo_dir_msg='nothing to get from %s', noinfo_file_msg='already present', action='get', logger=lgr, refds=refds_path): yield r @staticmethod def custom_result_summary_renderer(res): from datalad.ui import ui from os import linesep if not len(res): ui.message("Got nothing new") return nfiles = count_results(res, type='file') nsuccess_file = count_results(res, type='file', status='ok') nfailure = nfiles - nsuccess_file msg = "Tried to get %d %s that had no content yet." % ( nfiles, single_or_plural("file", "files", nfiles)) if nsuccess_file: msg += " Successfully obtained %d. " % nsuccess_file if nfailure: msg += " %d (failed)." % (nfailure, ) ui.message(msg) # if just a few or less than initially explicitly requested if len(res) < 10: msg = linesep.join([ "{path}{type} ... {suc}".format( suc=item.get('status'), path=item.get('path'), type=' [{}]'.format(item['type']) if 'type' in item else '') for item in res ]) ui.message(msg)
recursion_flag = Parameter( args=( "-r", "--recursive", ), action="store_true", doc="""if set, recurse into potential subdatasets""") recursion_limit = Parameter( args=( "-R", "--recursion-limit", ), metavar="LEVELS", constraints=EnsureInt() | EnsureNone(), doc="""limit recursion into subdatasets to the given number of levels""") contains = Parameter(args=('--contains', ), metavar='PATH', action='append', doc="""limit to the subdatasets containing the given path. If a root path of a subdataset is given, the last considered dataset will be the subdataset itself.[CMD: This option can be given multiple times CMD][PY: Can be a list with multiple paths PY], in which case datasets that contain any of the given paths will be considered.""", constraints=EnsureStr() | EnsureNone()) fulfilled = Parameter(args=("--fulfilled", ), doc="""DEPRECATED: use [CMD: --state CMD][PY: `state` PY]
class ExportToFigshare(Interface): """Export the content of a dataset as a ZIP archive to figshare Very quick and dirty approach. Ideally figshare should be supported as a proper git annex special remote. Unfortunately, figshare does not support having directories, and can store only a flat list of files. That makes it impossible for any sensible publishing of complete datasets. The only workaround is to publish dataset as a zip-ball, where the entire content is wrapped into a .zip archive for which figshare would provide a navigator. """ from datalad.support.param import Parameter from datalad.distribution.dataset import datasetmethod from datalad.interface.utils import eval_results from datalad.distribution.dataset import EnsureDataset from datalad.support.constraints import EnsureNone, EnsureInt, EnsureStr _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc=""""specify the dataset to export. If no dataset is given, an attempt is made to identify the dataset based on the current working directory.""", constraints=EnsureDataset() | EnsureNone()), filename=Parameter( args=("filename",), metavar="PATH", nargs='?', doc="""File name of the generated ZIP archive. If no file name is given the archive will be generated in the top directory of the dataset and will be named: datalad_<dataset_uuid>.zip.""", constraints=EnsureStr() | EnsureNone()), no_annex=Parameter( args=("--no-annex",), action="store_true", doc="""By default the generated .zip file would be added to annex, and all files would get registered in git-annex to be available from such a tarball. Also upon upload we will register for that archive to be a possible source for it in annex. Setting this flag disables this behavior."""), missing_content=Parameter( args=("--missing-content",), metavar="error|continue|ignore", doc="""By default, any discovered file with missing content will result in an error and the plugin is aborted. Setting this to 'continue' will issue warnings instead of failing on error. The value 'ignore' will only inform about problem at the 'debug' log level. The latter two can be helpful when generating a TAR archive from a dataset where some file content is not available locally.""", constraints=EnsureStr()), # article_id=Parameter( # args=("--project-id",), # metavar="ID", # doc="""If given, article (if article_id is not provided) will be # created in that project.""", # constraints=EnsureInt() | EnsureNone()), article_id=Parameter( args=("--article-id",), metavar="ID", doc="""Which article to publish to.""", constraints=EnsureInt() | EnsureNone()), ) @staticmethod @datasetmethod(name='export_to_figshare') @eval_results def __call__(dataset, filename=None, missing_content='error', no_annex=False, # TODO: support working with projects and articles within them # project_id=None, article_id=None): import os import logging lgr = logging.getLogger('datalad.plugin.export_to_figshare') from datalad.ui import ui from datalad.api import add_archive_content from datalad.api import export_archive from datalad.distribution.dataset import require_dataset from datalad.support.annexrepo import AnnexRepo dataset = require_dataset(dataset, check_installed=True, purpose='export to figshare') if not isinstance(dataset.repo, AnnexRepo): raise ValueError( "%s is not an annex repo, so annexification could be done" % dataset ) if dataset.repo.is_dirty(): raise RuntimeError( "Paranoid authors of DataLad refuse to proceed in a dirty repository" ) if filename is None: filename = dataset.path lgr.info( "Exporting current tree as an archive under %s since figshare " "does not support directories", filename ) archive_out = next( export_archive( dataset, filename=filename, archivetype='zip', missing_content=missing_content, return_type="generator" ) ) assert archive_out['status'] == 'ok' fname = archive_out['path'] lgr.info("Uploading %s to figshare", fname) figshare = FigshareRESTLaison() if not article_id: # TODO: ask if it should be an article within a project if ui.is_interactive: # or should we just upload to a new article? if ui.yesno( "Would you like to create a new article to upload to? " "If not - we will list existing articles", title="Article" ): article = figshare.create_article( title=os.path.basename(dataset.path) ) lgr.info( "Created a new (private) article %(id)s at %(url_private_html)s. " "Please visit it, enter additional meta-data and make public", article ) article_id = article['id'] else: article_id = int(ui.question( "Which of the articles should we upload to.", choices=list(map(str, figshare.get_article_ids())) )) if not article_id: raise ValueError("We need an article to upload to.") file_info = figshare.upload_file( fname, files_url='account/articles/%s/files' % article_id ) if no_annex: lgr.info("Removing generated tarball") unlink(fname) else: # I will leave all the complaining etc to the dataset add if path # is outside etc lgr.info("'Registering' %s within annex", fname) repo = dataset.repo repo.add(fname, git=False) key = repo.get_file_key(fname) lgr.info("Adding URL %(download_url)s for it", file_info) repo._annex_custom_command([], [ "git", "annex", "registerurl", '-c', 'annex.alwayscommit=false', key, file_info['download_url'] ] ) lgr.info("Registering links back for the content of the archive") add_archive_content( fname, annex=dataset.repo, delete_after=True, # just remove extracted into a temp dir allow_dirty=True, # since we have a tarball commit=False # we do not want to commit anything we have done here ) lgr.info("Removing generated and now registered in annex archive") repo.drop(key, key=True, options=['--force']) repo.remove(fname, force=True) # remove the tarball # if annex in {'delete'}: # dataset.repo.remove(fname) # else: # # kinda makes little sense I guess. # # Made more sense if export_archive could export an arbitrary treeish # # so we could create a branch where to dump and export to figshare # # (kinda closer to my idea) # dataset.save(fname, message="Added the entire dataset into a zip file") # TODO: add to downloader knowledge about figshare token so it could download-url # those zipballs before they go public yield dict( status='ok', # TODO: add article url (which needs to be queried if only ID is known message="Published archive {}".format( file_info['download_url']), file_info=file_info, path=dataset, action='export_to_figshare', logger=lgr )
'destination': 'local', 'type': EnsureBool(), 'default': False, }, 'datalad.fake-dates-start': { 'ui': ('question', { 'title': 'Initial fake date', 'text': 'When faking dates and there are no commits in any local branches, generate the date by adding one second to this value (Unix epoch time). The value must be positive.' }), 'type': EnsureInt(), 'default': 1112911993, }, 'datalad.github.token-note': { 'ui': ('question', { 'title': 'Github token note', 'text': 'Description for a Personal access token to generate.' }), 'default': 'DataLad', }, 'datalad.tests.nonetwork': { 'ui': ('yesno', { 'title': 'Skips network tests completely if this flag is set Examples include test for s3, git_repositories, openfmri etc'
class Get(Interface): """Get any dataset content (files/directories/subdatasets). This command only operates on dataset content. To obtain a new independent dataset from some source use the `install` command. By default this command operates recursively within a dataset, but not across potential subdatasets, i.e. if a directory is provided, all files in the directory are obtained. Recursion into subdatasets is supported too. If enabled, relevant subdatasets are detected and installed in order to fulfill a request. Known data locations for each requested file are evaluated and data are obtained from some available location (according to git-annex configuration and possibly assigned remote priorities), unless a specific source is specified. .. note:: Power-user info: This command uses :command:`git annex get` to fulfill file handles. """ _examples_ = [ dict(text="Get a single file", code_py="get('path/to/file')", code_cmd="datalad get <path/to/file>"), dict(text="Get contents of a directory", code_py="get('path/to/dir/')", code_cmd="datalad get <path/to/dir/>"), dict( text="Get all contents of the current dataset and its subdatasets", code_py="get(dataset='.', recursive=True)", code_cmd="datalad get . -r"), dict( text="Get (clone) a registered subdataset, but don't retrieve data", code_py="get('path/to/subds', get_data=False)", code_cmd="datalad get -n <path/to/subds>"), ] _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar="PATH", doc="""specify the dataset to perform the add operation on, in which case `path` arguments are interpreted as being relative to this dataset. If no dataset is given, an attempt is made to identify a dataset for each input `path`""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path", ), metavar="PATH", doc="""path/name of the requested dataset component. The component must already be known to a dataset. To add new components to a dataset use the `add` command""", nargs="*", constraints=EnsureStr() | EnsureNone()), source=Parameter( args=( "-s", "--source", ), metavar="LABEL", doc="""label of the data source to be used to fulfill requests. This can be the name of a dataset :term:`sibling` or another known source""", constraints=EnsureStr() | EnsureNone()), recursive=recursion_flag, recursion_limit=Parameter( args=( "-R", "--recursion-limit", ), metavar="LEVELS", constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(), doc="""limit recursion into subdataset to the given number of levels. Alternatively, 'existing' will limit recursion to subdatasets that already existed on the filesystem at the start of processing, and prevent new subdatasets from being obtained recursively."""), get_data=Parameter( args=( "-n", "--no-data", ), dest='get_data', action='store_false', doc= """whether to obtain data for all file handles. If disabled, `get` operations are limited to dataset handles.[CMD: This option prevents data for file handles from being obtained CMD]"""), description=location_description, reckless=reckless_opt, jobs=jobs_opt) @staticmethod @datasetmethod(name='get') @eval_results def __call__( path=None, source=None, dataset=None, recursive=False, recursion_limit=None, get_data=True, description=None, reckless=None, jobs='auto', ): refds_path = Interface.get_refds_path(dataset) if not (dataset or path): raise InsufficientArgumentsError( "Neither dataset nor target path(s) provided") if dataset and not path: # act on the whole dataset if nothing else was specified path = refds_path # we have to have a single dataset to operate on refds = require_dataset(dataset, check_installed=True, purpose='get content') content_by_ds = {} # use subdatasets() to discover any relevant content that is not # already present in the root dataset (refds) for sdsres in Subdatasets.__call__( contains=path, # maintain path argument semantics and pass in dataset arg # as is dataset=dataset, # always come from the top to get sensible generator behavior bottomup=False, # when paths are given, they will constrain the recursion # automatically, and we need to enable recursion so we can # location path in subdatasets several levels down recursive=True if path else recursive, recursion_limit=None if path else recursion_limit, return_type='generator', on_failure='ignore'): if sdsres.get('type', None) != 'dataset': # if it is not about a 'dataset' it is likely content in # the root dataset if sdsres.get('status', None) == 'impossible' and \ sdsres.get('message', None) == \ 'path not contained in any matching subdataset': target_path = Path(sdsres['path']) if refds.pathobj != target_path and \ refds.pathobj not in target_path.parents: yield dict( action='get', path=str(target_path), status='error', message=('path not associated with dataset %s', refds), ) continue # check if we need to obtain anything underneath this path # the subdataset() call above will only look _until_ it # hits the targetpath for res in _install_targetpath( refds, Path(sdsres['path']), recursive, recursion_limit, reckless, refds_path, description, jobs=jobs, ): # fish out the datasets that 'contains' a targetpath # and store them for later if res.get('status', None) in ('ok', 'notneeded') and \ 'contains' in res: dsrec = content_by_ds.get(res['path'], set()) dsrec.update(res['contains']) content_by_ds[res['path']] = dsrec if res.get('status', None) != 'notneeded': # all those messages on not having installed anything # are a bit pointless # "notneeded" for annex get comes below yield res else: # dunno what this is, send upstairs yield sdsres # must continue for both conditional branches above # the rest is about stuff in real subdatasets continue # instance of the closest existing dataset for this result ds = Dataset(sdsres['parentds'] if sdsres.get('state', None) == 'absent' else sdsres['path']) assert 'contains' in sdsres # explore the unknown for target_path in sdsres.get('contains', []): # essentially the same as done above for paths in the root # dataset, but here we are starting from the closest # discovered subdataset for res in _install_targetpath( ds, Path(target_path), recursive, recursion_limit, reckless, refds_path, description, jobs=jobs, ): known_ds = res['path'] in content_by_ds if res.get('status', None) in ('ok', 'notneeded') and \ 'contains' in res: dsrec = content_by_ds.get(res['path'], set()) dsrec.update(res['contains']) content_by_ds[res['path']] = dsrec # prevent double-reporting of datasets that have been # installed by explorative installation to get to target # paths, prior in this loop if res.get('status', None) != 'notneeded' or not known_ds: yield res if not get_data: # done already return # and now annex-get, this could all be done in parallel now for ds, content in content_by_ds.items(): for res in _get_targetpaths(Dataset(ds), content, refds.path, source, jobs): if res['path'] not in content_by_ds: # we had reports on datasets and subdatasets already # before the annex stage yield res
class Search(Interface): """Search a dataset's metadata. Search capabilities depend on the amount and nature of metadata available in a dataset. This can include metadata about a dataset as a whole, or metadata on dataset content (e.g. one or more files). One dataset can also contain metadata from multiple subdatasets (see the 'aggregate-metadata' command), in which case a search can discover any dataset or any file in of these datasets. A search index is automatically built from the available metadata of any dataset or file, and a schema for this index is generated dynamically, too. Consequently, the search index will be tailored to data provided in a particular collection of datasets. Metadata fields (and possibly also values) are typically defined terms from a controlled vocabulary. Term definitions are accessible via the --show-keys flag. DataLad's search is built on the Python package 'Whoosh', which provides a powerful query language. Links to a description of the language and particular feature can be found below. Here are a few examples. Basic search:: % datalad search searchterm Search for a file:: % datalad search searchterm type:file Show definitions of search keys/fields:: % datalad search --show-keys @id (unique identifier of an entity) dcterms:rights (http://purl.org/dc/terms/rights) duration(s) {'unit (http://purl.obolibrary.org/obo/UO_0000000)': 'http://purl.obolibrary.org/obo/UO_0000010', '@id': 'https://www.w3.org/TR/owl-time/#Duration'} name (http://schema.org/name) ... *Performance considerations* For dataset collections with many files (100k+) generating a comprehensive search index comprised of documents for datasets and individual files can take a considerable amount of time. If this becomes an issue, search index generation can be limited to a particular type of document (see the 'metadata --reporton' option for possible values). The configuration setting 'datalad.metadata.searchindex-documenttype' will be queried on search index generation. It is recommended to place an appropriate configuration into a dataset's configuration file (.datalad/config):: [datalad "metadata"] searchindex-documenttype = datasets .. seealso:: - Description of the Whoosh query language: http://whoosh.readthedocs.io/en/latest/querylang.html) - Description of a number of query language customizations that are enabled in DataLad, such as, querying multiple fields by default and fuzzy term matching: http://whoosh.readthedocs.io/en/latest/parsing.html#common-customizations """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to perform the query operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), query=Parameter( args=("query", ), metavar='QUERY', nargs="*", doc="""search query using the Whoosh query language (see link to detailed description above). For simple queries, any number of search terms can be given as a list[CMD: (space-separated) CMD], and the query will return all hits that match all terms (AND) in any combination of fields (OR)."""), force_reindex=Parameter( args=("--reindex", ), dest='force_reindex', action='store_true', doc="""force rebuilding the search index, even if no change in the dataset's state has been detected. This is mostly useful for developing new metadata support extensions."""), max_nresults=Parameter( args=("--max-nresults", ), doc="""maxmimum number of search results to report. Setting this to 0 will report all search matches, and make searching substantially slower on large metadata sets.""", constraints=EnsureInt()), show_keys=Parameter( args=('--show-keys', ), action='store_true', doc="""if given, a list of known search keys is shown (one per line). No other action is performed (except for reindexing), even if other arguments are given. Each key is accompanied by a term definition in parenthesis. In most cases a definition is given in the form of a URL. If an ontology definition for a term is known, this URL can resolve to a webpage that provides a comprehensive definition of the term. However, for speed reasons term resolution is solely done on information contained in a local dataset's metadata, and definition URLs might be outdated or point to no longer existing resources.""" ), show_query=Parameter( args=('--show-query', ), action='store_true', doc="""if given, the formal query that was generated from the given query string is shown, but not actually executed. This is mostly useful for debugging purposes."""), ) @staticmethod @datasetmethod(name='search') @eval_results def __call__(query=None, dataset=None, force_reindex=False, max_nresults=20, show_keys=False, show_query=False): from whoosh import qparser as qparse try: ds = require_dataset(dataset, check_installed=True, purpose='dataset search') if ds.id is None: raise NoDatasetArgumentFound( "This does not seem to be a dataset (no DataLad dataset ID " "found). 'datalad create --force %s' can initialize " "this repository as a DataLad dataset" % ds.path) except NoDatasetArgumentFound: for r in _search_from_virgin_install(dataset, query): yield r return # where does the bunny have the eggs? index_dir = opj(ds.path, get_git_dir(ds.path), 'datalad', 'search_index') idx_obj = _get_search_index(index_dir, ds, force_reindex) if show_keys: definitions_fname = opj(index_dir, 'datalad_term_definitions.json.gz') try: defs = jsonload(gzopen(definitions_fname)) except Exception as e: lgr.warning( 'No term definitions found alongside search index: %s', exc_str(e)) defs = {} for k in idx_obj.schema.names(): print('{}{}'.format( k, ' {}'.format(defs[k] if isinstance(defs[k], dict) else '({})'.format(defs[k])) if k in defs else '')) return if not query: return with idx_obj.searcher() as searcher: # parse the query string, default whoosh parser ATM, could be # tailored with plugins parser = qparse.MultifieldParser(idx_obj.schema.names(), idx_obj.schema) # XXX: plugin is broken in Debian's whoosh 2.7.0-2, but already fixed # upstream parser.add_plugin(qparse.FuzzyTermPlugin()) parser.add_plugin(qparse.GtLtPlugin()) # replace field defintion to allow for colons to be part of a field's name: parser.replace_plugin( qparse.FieldsPlugin(expr=r"(?P<text>[()<>:\w]+|[*]):")) # for convenience we accept any number of args-words from the # shell and put them together to a single string here querystr = ' '.join(assure_list(query)) # this gives a formal whoosh query wquery = parser.parse(querystr) if show_query: print(wquery) return # perform the actual search hits = searcher.search( wquery, terms=True, limit=max_nresults if max_nresults > 0 else None) # cheap way to get an approximate number of hits, without an expensive # scoring of all items # disabled: unreliable estimate, often confusing #nhits = hits.estimated_min_length() # report query stats topstr = '{} top {}'.format( max_nresults, single_or_plural('match', 'matches', max_nresults)) lgr.info('Query completed in {} sec.{}'.format( hits.runtime, ' Reporting {}.'.format(( 'up to ' + topstr) if max_nresults > 0 else 'all matches') if not hits.is_empty() else ' No matches.')) if not hits: return nhits = 0 for hit in hits: res = dict( action='search', status='ok', logger=lgr, refds=ds.path, # normpath to avoid trailing dot path=normpath(opj(ds.path, hit['path'])), query_matched={ assure_unicode(k): assure_unicode(v) if isinstance( v, unicode_srctypes) else v for k, v in hit.matched_terms() }, metadata={ k: v for k, v in hit.fields().items() if k not in ('path', 'parentds') }) if 'parentds' in hit: res['parentds'] = normpath(opj(ds.path, hit['parentds'])) yield res nhits += 1 if max_nresults and nhits == max_nresults: lgr.info("Reached the limit of {}, there could be more which " "were not reported.".format(topstr))
'ui': ('question', { 'title': 'This flag is used by the datalad extract_tb function which extracts and formats stack-traces. It caps the number of lines to DATALAD_EXC_STR_TBLIMIT of pre-processed entries from traceback.'}), }, 'datalad.fake-dates': { 'ui': ('yesno', { 'title': 'Fake (anonymize) dates', 'text': 'Should the dates in the logs be faked?'}), 'destination': 'local', 'type': EnsureBool(), 'default': False, }, 'datalad.fake-dates-start': { 'ui': ('question', { 'title': 'Initial fake date', 'text': 'When faking dates and there are no commits in any local branches, generate the date by adding one second to this value (Unix epoch time). The value must be positive.'}), 'type': EnsureInt(), 'default': 1112911993, }, 'datalad.tests.nonetwork': { 'ui': ('yesno', { 'title': 'Skips network tests completely if this flag is set Examples include test for s3, git_repositories, openfmri etc'}), 'type': EnsureBool(), }, 'datalad.tests.nonlo': { 'ui': ('question', { 'title': 'Specifies network interfaces to bring down/up for testing. Currently used by travis.'}), }, 'datalad.tests.noteardown': { 'ui': ('yesno', { 'title': 'Does not execute teardown_package which cleans up temp files and directories created by tests if this flag is set'}), 'type': EnsureBool(),
class Get(Interface): """Get any dataset content (files/directories/subdatasets). This command only operates on dataset content. To obtain a new independent dataset from some source use the `install` command. By default this command operates recursively within a dataset, but not across potential subdatasets, i.e. if a directory is provided, all files in the directory are obtained. Recursion into subdatasets is supported too. If enabled, relevant subdatasets are detected and installed in order to fulfill a request. Known data locations for each requested file are evaluated and data are obtained from some available location (according to git-annex configuration and possibly assigned remote priorities), unless a specific source is specified. .. note:: Power-user info: This command uses :command:`git annex get` to fulfill file handles. """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), metavar="PATH", doc="""specify the dataset to perform the add operation on, in which case `path` arguments are interpreted as being relative to this dataset. If no dataset is given, an attempt is made to identify a dataset for each input `path`""", constraints=EnsureDataset() | EnsureNone()), path=Parameter( args=("path", ), metavar="PATH", doc="""path/name of the requested dataset component. The component must already be known to a dataset. To add new components to a dataset use the `add` command""", nargs="*", constraints=EnsureStr() | EnsureNone()), source=Parameter( args=( "-s", "--source", ), metavar="LABEL", doc="""label of the data source to be used to fulfill requests. This can be the name of a dataset :term:`sibling` or another known source""", constraints=EnsureStr() | EnsureNone()), recursive=recursion_flag, recursion_limit=Parameter( args=("--recursion-limit", ), metavar="LEVELS", constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(), doc="""limit recursion into subdataset to the given number of levels. Alternatively, 'existing' will limit recursion to subdatasets that already existed on the filesystem at the start of processing, and prevent new subdatasets from being obtained recursively."""), get_data=Parameter( args=( "-n", "--no-data", ), dest='get_data', action='store_false', doc= """whether to obtain data for all file handles. If disabled, `get` operations are limited to dataset handles.[CMD: This option prevents data for file handles from being obtained CMD]"""), reckless=reckless_opt, git_opts=git_opts, annex_opts=annex_opts, annex_get_opts=annex_get_opts, jobs=jobs_opt, verbose=verbose) # Note: May be use 'git annex find --not --in here' to have a list of all # files to actually get and give kind of a progress in terms of number # files processed ... @staticmethod @datasetmethod(name='get') def __call__( path, source=None, dataset=None, recursive=False, recursion_limit=None, get_data=True, reckless=False, git_opts=None, annex_opts=None, annex_get_opts=None, jobs=None, verbose=False, # internal -- instead of returning 'get'ed items, return final # content_by_ds, unavailable_paths. To be used by the call from # Install.__call__ and done so to avoid creating another reusable # function which would need to duplicate all this heavy list of # kwargs _return_datasets=False): # IMPLEMENTATION CONCEPT: # # 1. turn all input paths into absolute paths # 2. Sort the world into existing handles and the rest # 3. Try locate missing handles (obtain subdatasets along the way) # 4. Expand into subdatasets with recursion enables (potentially # obtain even more subdatasets # 5. Shoot info of which handles to get in each subdataset to, # git-annex, once at the very end # TODO: consider allowing an empty `path` argument, as with other commands, # to indicate CWD resolved_paths, dataset_path = get_normalized_path_arguments( path, dataset, default=None) if not resolved_paths: raise InsufficientArgumentsError( "`get` needs at least one path as argument") # sort paths into the respective datasets dir_lookup = {} content_by_ds, unavailable_paths, nondataset_paths = \ get_paths_by_dataset(resolved_paths, recursive=recursive, recursion_limit=recursion_limit, dir_lookup=dir_lookup) lgr.debug( "Found %i existing dataset(s) to get content in " "and %d unavailable paths", len(content_by_ds), len(unavailable_paths)) # IMPORTANT NOTE re `content_by_ds` # each key is a subdataset that we need to get something in # if the value[0] is the subdataset's path, we want all of it # if the value[0] == curdir, we just installed it as part of # resolving file handles and we did not say anything but "give # me the dataset handle" # explore the unknown for path in sorted(unavailable_paths): # how close can we get? dspath = GitRepo.get_toppath(path) if dspath is None: # nothing we can do for this path continue ds = Dataset(dspath) # must always yield a dataset -- we sorted out the ones outside # any dataset at the very top assert ds.is_installed() # now actually obtain whatever is necessary to get to this path containing_ds = install_necessary_subdatasets(ds, path, reckless) if containing_ds.path != ds.path: lgr.debug( "Installed %s to fulfill request for content for " "path %s", containing_ds, path) # mark resulting dataset as auto-installed if containing_ds.path == path: # we had to get the entire dataset, not something within # mark that it just appeared content_by_ds[path] = [curdir] else: # we need to get content within content_by_ds[path] = [path] if recursive and not recursion_limit == 'existing': # obtain any subdatasets underneath the paths given inside the # subdatasets that we know already exist # unless we do not want recursion into not-yet-installed datasets for subdspath in sorted(content_by_ds.keys()): for content_path in content_by_ds[subdspath]: if not isdir(content_path): # a non-directory cannot have content underneath continue subds = Dataset(subdspath) lgr.info( "Obtaining %s %s recursively", subds, ("underneath %s" % content_path if subds.path != content_path else "")) cbysubds = _recursive_install_subds_underneath( subds, # `content_path` was explicitly given as input # we count recursions from the input, hence we # can start with the full number recursion_limit, reckless, # protect against magic marker misinterpretation # only relevant for _get, hence replace here start=content_path if content_path != curdir else None) # gets file content for all freshly installed subdatasets content_by_ds.update(cbysubds) ## we have now done everything we could to obtain whatever subdataset ## to get something on the file system for previously unavailable paths ## check and sort one last content_by_ds, unavailable_paths, nondataset_paths2 = \ get_paths_by_dataset( unavailable_paths, recursive=recursive, recursion_limit=recursion_limit, out=content_by_ds, dir_lookup=dir_lookup) nondataset_paths.extend(nondataset_paths2) if nondataset_paths: lgr.warning("ignored paths that do not belong to any dataset: %s", nondataset_paths) if unavailable_paths: lgr.warning('ignored non-existing paths: %s', unavailable_paths) # hand over to git-annex results = list( chain.from_iterable( _get(content_by_ds, refpath=dataset_path, source=source, jobs=jobs, get_data=get_data))) # ??? should we in _return_datasets case just return both content_by_ds # and unavailable_paths may be so we provide consistent across runs output # and then issue outside similar IncompleteResultsError? if unavailable_paths: # and likely other error flags if _return_datasets: results = sorted( set(content_by_ds).difference(unavailable_paths)) raise IncompleteResultsError(results, failed=unavailable_paths) else: return sorted(content_by_ds) if _return_datasets else results @staticmethod def result_renderer_cmdline(res, args): from datalad.ui import ui from os import linesep if res is None: res = [] if not isinstance(res, list): res = [res] if not len(res): ui.message("Got nothing new") return # provide summary nsuccess = sum( item.get('success', False) if isinstance(item, dict) else True for item in res) nfailure = len(res) - nsuccess msg = "Tried to get %d %s." % ( len(res), single_or_plural("file", "files", len(res))) if nsuccess: msg += " Got %d. " % nsuccess if nfailure: msg += " Failed to get %d." % (nfailure, ) ui.message(msg) # if just a few or less than initially explicitly requested if len(res) < 10 or args.verbose: msg = linesep.join([ "{path} ... {suc}".format( suc="ok." if isinstance(item, Dataset) or item.get('success', False) else "failed. (%s)" % item.get('note', 'unknown reason'), path=item.get('file') if isinstance(item, dict) else item.path) for item in res ]) ui.message(msg)
class Search(Interface): """Search a dataset's metadata. Search capabilities depend on the amount and nature of metadata available in a dataset. This can include metadata about a dataset as a whole, or metadata on dataset content (e.g. one or more files). One dataset can also contain metadata from multiple subdatasets (see the 'aggregate-metadata' command), in which case a search can discover any dataset or any file in of these datasets. *Search modes* WRITE ME A search index is automatically built from the available metadata of any dataset or file, and a schema for this index is generated dynamically, too. Consequently, the search index will be tailored to data provided in a particular collection of datasets. Metadata fields (and possibly also values) are typically defined terms from a controlled vocabulary. Field names are accessible via the --show-keys flag. DataLad's search is built on the Python package 'Whoosh', which provides a powerful query language. Links to a description of the language and particular feature can be found below. Here are a few examples. Basic search:: % datalad search searchterm Search for a file:: % datalad search searchterm type:file *Performance considerations* For dataset collections with many files (100k+) generating a comprehensive search index comprised of documents for datasets and individual files can take a considerable amount of time. If this becomes an issue, search index generation can be limited to a particular type of document (see the 'metadata --reporton' option for possible values). The per-mode configuration setting 'datalad.search.index-<mode>-documenttype' will be queried on search index generation. It is recommended to place an appropriate configuration into a dataset's configuration file (.datalad/config):: [datalad "search"] index-default-documenttype = datasets .. seealso:: - Description of the Whoosh query language: http://whoosh.readthedocs.io/en/latest/querylang.html) - Description of a number of query language customizations that are enabled in DataLad, such as, querying multiple fields by default and fuzzy term matching: http://whoosh.readthedocs.io/en/latest/parsing.html#common-customizations """ _params_ = dict( dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to perform the query operation on. If no dataset is given, an attempt is made to identify the dataset based on the current working directory and/or the `path` given""", constraints=EnsureDataset() | EnsureNone()), query=Parameter( args=("query",), metavar='QUERY', nargs="*", doc="""search query using the Whoosh query language (see link to detailed description above). For simple queries, any number of search terms can be given as a list[CMD: (space-separated) CMD], and the query will return all hits that match all terms (AND) in any combination of fields (OR)."""), force_reindex=Parameter( args=("--reindex",), dest='force_reindex', action='store_true', doc="""force rebuilding the search index, even if no change in the dataset's state has been detected. This is mostly useful for developing new metadata support extensions."""), max_nresults=Parameter( args=("--max-nresults",), doc="""maxmimum number of search results to report. Setting this to 0 will report all search matches, and make searching substantially slower on large metadata sets.""", constraints=EnsureInt()), mode=Parameter( args=("--mode",), choices=('egrep', 'textblob', 'autofield'), doc="""Mode of search index structure and content. See section SEARCH MODES for details. """), show_keys=Parameter( args=('--show-keys',), action='store_true', doc="""if given, a list of known search keys is shown (one per line). No other action is performed (except for reindexing), even if other arguments are given. Each key is accompanied by a term definition in parenthesis. In most cases a definition is given in the form of a URL. If an ontology definition for a term is known, this URL can resolve to a webpage that provides a comprehensive definition of the term. However, for speed reasons term resolution is solely done on information contained in a local dataset's metadata, and definition URLs might be outdated or point to no longer existing resources."""), show_query=Parameter( args=('--show-query',), action='store_true', doc="""if given, the formal query that was generated from the given query string is shown, but not actually executed. This is mostly useful for debugging purposes."""), ) @staticmethod @datasetmethod(name='search') @eval_results def __call__(query=None, dataset=None, force_reindex=False, max_nresults=20, mode=None, show_keys=False, show_query=False): try: ds = require_dataset(dataset, check_installed=True, purpose='dataset search') if ds.id is None: raise NoDatasetArgumentFound( "This does not seem to be a dataset (no DataLad dataset ID " "found). 'datalad create --force %s' can initialize " "this repository as a DataLad dataset" % ds.path) except NoDatasetArgumentFound: for r in _search_from_virgin_install(dataset, query): yield r return if mode is None: # let's get inspired by what the dataset/user think is # default mode = ds.config.obtain('datalad.search.default-mode') if mode == 'egrep': searcher = _EGrepSearch elif mode == 'textblob': searcher = _BlobSearch elif mode == 'autofield': searcher = _AutofieldSearch else: raise ValueError( 'unknown search mode "{}"'.format(mode)) searcher = searcher(ds, force_reindex=force_reindex) if show_keys: searcher.show_keys() return if not query: return if show_query: print(repr(searcher.get_query(query))) return for r in searcher( query, max_nresults=max_nresults): yield r
class HTCResults(Interface): """TODO """ # make the custom renderer the default one, as the global default renderer # does not yield meaningful output for this command result_renderer = 'tailored' _params_ = dict( cmd=Parameter(args=("cmd", ), metavar=("SUBCOMMAND", ), nargs='?', doc="""""", constraints=EnsureChoice('list', 'merge', 'remove')), dataset=Parameter( args=("-d", "--dataset"), doc="""specify the dataset to record the command results in. An attempt is made to identify the dataset based on the current working directory. If a dataset is given, the command will be executed in the root directory of this dataset.""", constraints=EnsureDataset() | EnsureNone()), submission=Parameter(args=("submission", ), nargs='?', metavar='SUBMISSION', doc=""""""), job=Parameter(args=( "-j", "--job", ), metavar='NUMBER', doc="""""", constraints=EnsureInt() | EnsureNone()), all=Parameter(args=("--all", ), action='store_true', doc=""""""), ) @staticmethod @datasetmethod(name='htc_results') @eval_results def __call__(cmd='list', dataset=None, submission=None, job=None, all=False): ds = require_dataset( dataset, check_installed=True, purpose='handling results of remote command executions') if cmd == 'list': jw = _list_job sw = _list_submission elif cmd == 'merge': jw = _apply_output sw = None elif cmd == 'remove': if not all and not submission and not job: raise ValueError( "use the '--all' flag to remove all results across all " "submissions") jw = _remove_dir sw = _remove_dir else: raise ValueError("unknown sub-command '{}'".format(cmd)) for res in _doit(ds, submission, job, jw, sw): yield res @staticmethod def custom_result_renderer(res, **kwargs): # pragma: no cover from datalad.ui import ui if not res['status'] == 'ok' or not res['action'].startswith('htc_'): # logging reported already return action = res['action'].split('_')[-1] ui.message('{action} {sub}{job}{state}{cmd}'.format( action=ac.color_word(action, kw_color_map.get(action, ac.WHITE)) if action != 'list' else '', sub=res['submission'], job=' :{}'.format(res['job']) if 'job' in res else '', state=' [{}]'.format( ac.color_word(res['state'], kw_color_map.get(res['state'], ac.MAGENTA) ) if res.get('state', None) else 'unknown') if action == 'list' else '', cmd=': {}'.format(_format_cmd_shorty(res['cmd'])) if 'cmd' in res else '', ))