class CreateTestDataset(Interface):
"""Create test (meta-)dataset.
"""
# XXX prevent common args from being added to the docstring
_no_eval_results = True
_params_ = dict(
path=Parameter(
args=("path",),
doc="path/name where to create (if specified, must not exist)",
constraints=EnsureStr() | EnsureNone()),
spec=Parameter(
args=("--spec",),
doc="""\
spec for hierarchy, defined as a min-max (min could be omitted to assume 0)
defining how many (random number from min to max) of sub-datasets to generate
at any given level of the hierarchy. Each level separated from each other with /.
Example: 1-3/-2 would generate from 1 to 3 subdatasets at the top level, and
up to two within those at the 2nd level
""",
constraints=EnsureStr() | EnsureNone()),
seed=Parameter(
args=("--seed",),
doc="""seed for rng""",
constraints=EnsureInt() | EnsureNone()),
)
@staticmethod
def __call__(path=None, spec=None, seed=None):
levels = _parse_spec(spec)
if seed is not None:
# TODO: if to be used within a bigger project we shouldn't seed main RNG
random.seed(seed)
if path is None:
kw = get_tempfile_kwargs({}, prefix="ds")
path = tempfile.mkdtemp(**kw)
else:
# so we don't override anything
assert not exists(path)
os.makedirs(path)
# now we should just make it happen and return list of all the datasets
return list(_makeds(path, levels))
@staticmethod
def result_renderer_cmdline(res, args):
from datalad.ui import ui
if res is None:
res = []
if not len(res):
ui.message("No repos were created... oops")
return
items = '\n'.join(map(str, res))
msg = "{n} installed {obj} available at\n{items}".format(
obj='items are' if len(res) > 1 else 'item is',
n=len(res),
items=items)
ui.message(msg)
args=("-D", "--description",),
constraints=EnsureStr() | EnsureNone(),
doc="""short description to use for a dataset location. Its primary
purpose is to help humans to identify a dataset copy (e.g., "mike's dataset
on lab server"). Note that when a dataset is published, this information
becomes available on the remote side.""")
recursion_flag = Parameter(
args=("-r", "--recursive",),
action="store_true",
doc="""if set, recurse into potential subdataset""")
recursion_limit = Parameter(
args=("--recursion-limit",),
metavar="LEVELS",
constraints=EnsureInt() | EnsureNone(),
doc="""limit recursion into subdataset to the given number of levels""")
shared_access_opt = Parameter(
args=('--shared-access',),
metavar='MODE',
doc="""configure shared access to a dataset, see `git init --shared`
documentation for complete details on the supported scenarios. Possible
values include: 'false', 'true', 'group', and 'all'""")
super_datasets_flag = Parameter(
args=("-S", "--super-datasets",),
action="store_true",
doc="""if set, save a change in a dataset also in its superdataset""")
git_opts = Parameter(
class Search(Interface):
"""Search dataset metadata
DataLad can search metadata extracted from a dataset and/or aggregated into
a superdataset (see the `aggregate-metadata` command). This makes it
possible to discover datasets, or individual files in a dataset even when
they are not available locally.
Ultimately DataLad metadata are a graph of linked data structures. However,
this command does not (yet) support queries that can exploit all
information stored in the metadata. At the moment the following search
modes are implemented that represent different trade-offs between the
expressiveness of a query and the computational and storage resources
required to execute a query.
- egrep (default)
- egrepcs [case-sensitive egrep]
- textblob
- autofield
An alternative default mode can be configured by tuning the
configuration variable 'datalad.search.default-mode'::
[datalad "search"]
default-mode = egrepcs
Each search mode has its own default configuration for what kind of
documents to query. The respective default can be changed via configuration
variables::
[datalad "search"]
index-<mode_name>-documenttype = (all|datasets|files)
*Mode: egrep/egrepcs*
These search modes are largely ignorant of the metadata structure, and
simply perform matching of a search pattern against a flat
string-representation of metadata. This is advantageous when the query is
simple and the metadata structure is irrelevant, or precisely known.
Moreover, it does not require a search index, hence results can be reported
without an initial latency for building a search index when the underlying
metadata has changed (e.g. due to a dataset update). By default, these
search modes only consider datasets and do not investigate records for
individual files for speed reasons. Search results are reported in the
order in which they were discovered.
Queries can make use of Python regular expression syntax
(https://docs.python.org/3/library/re.html). In `egrep` mode, matching is
case-insensitive when the query does not contain upper case characters, but
is case-sensitive when it does. In `egrepcs` mode, matching is always
case-sensitive. Expressions will match anywhere in a metadata string, not
only at the start.
When multiple queries are given, all queries have to match for a search hit
(AND behavior).
It is possible to search individual metadata key/value items by prefixing
the query with a metadata key name, separated by a colon (':'). The key
name can also be a regular expression to match multiple keys. A query match
happens when any value of an item with a matching key name matches the query
(OR behavior). See examples for more information.
Examples:
Query for (what happens to be) an author::
% datalad search haxby
Queries are case-INsensitive when the query contains no upper case characters,
and can be regular expressions. Use `egrepcs` mode when it is desired
to perform a case-sensitive lowercase match::
% datalad search --mode egrepcs halchenko.*haxby
This search mode performs NO analysis of the metadata content. Therefore
queries can easily fail to match. For example, the above query implicitly
assumes that authors are listed in alphabetical order. If that is the
case (which may or may not be true), the following query would yield NO
hits::
% datalad search Haxby.*Halchenko
The ``textblob`` search mode represents an alternative that is more
robust in such cases.
For more complex queries multiple query expressions can be provided that
all have to match to be considered a hit (AND behavior). This query
discovers all files (non-default behavior) that match 'bids.type=T1w'
AND 'nifti1.qform_code=scanner'::
% datalad -c datalad.search.index-egrep-documenttype=all search bids.type:T1w nifti1.qform_code:scanner
Key name selectors can also be expressions, which can be used to select
multiple keys or construct "fuzzy" queries. In such cases a query matches
when any item with a matching key matches the query (OR behavior).
However, multiple queries are always evaluated using an AND conjunction.
The following query extends the example above to match any files that
have either 'nifti1.qform_code=scanner' or 'nifti1.sform_code=scanner'::
% datalad -c datalad.search.index-egrep-documenttype=all search bids.type:T1w nifti1.(q|s)form_code:scanner
*Mode: textblob*
This search mode is very similar to the ``egrep`` mode, but with a few key
differences. A search index is built from the string-representation of
metadata records. By default, only datasets are included in this index, hence
the indexing is usually completed within a few seconds, even for hundreds
of datasets. This mode uses its own query language (not regular expressions)
that is similar to other search engines. It supports logical conjunctions
and fuzzy search terms. More information on this is available from the Whoosh
project (search engine implementation):
- Description of the Whoosh query language:
http://whoosh.readthedocs.io/en/latest/querylang.html)
- Description of a number of query language customizations that are
enabled in DataLad, such as, fuzzy term matching:
http://whoosh.readthedocs.io/en/latest/parsing.html#common-customizations
Importantly, search hits are scored and reported in order of descending
relevance, hence limiting the number of search results is more meaningful
than in the 'egrep' mode and can also reduce the query duration.
Examples:
Search for (what happens to be) two authors, regardless of the order in
which those names appear in the metadata::
% datalad search --mode textblob halchenko haxby
Fuzzy search when you only have an approximate idea what you are looking
for or how it is spelled::
% datalad search --mode textblob haxbi~
Very fuzzy search, when you are basically only confident about the first
two characters and how it sounds approximately (or more precisely: allow
for three edits and require matching of the first two characters)::
% datalad search --mode textblob haksbi~3/2
Combine fuzzy search with logical constructs::
% datalad search --mode textblob 'haxbi~ AND (hanke OR halchenko)'
*Mode: autofield*
This mode is similar to the 'textblob' mode, but builds a vastly more
detailed search index that represents individual metadata variables as
individual fields. By default, this search index includes records for
datasets and individual fields, hence it can grow very quickly into
a huge structure that can easily take an hour or more to build and require
more than a GB of storage. However, limiting it to documents on datasets
(see above) retains the enhanced expressiveness of queries while
dramatically reducing the resource demands.
Examples:
List names of search index fields (auto-discovered from the set of
indexed datasets)::
% datalad search --mode autofield --show-keys name
Fuzzy search for datasets with an author that is specified in a particular
metadata field::
% datalad search --mode autofield bids.author:haxbi~ type:dataset
Search for individual files that carry a particular description
prefix in their 'nifti1' metadata::
% datalad search --mode autofield nifti1.description:FSL* type:file
*Reporting*
Search hits are returned as standard DataLad results. On the command line
the '--output-format' (or '-f') option can be used to tweak results for
further processing.
Examples:
Format search hits as a JSON stream (one hit per line)::
% datalad -f json search haxby
Custom formatting: which terms matched the query of particular
results. Useful for investigating fuzzy search results::
$ datalad -f '{path}: {query_matched}' search --mode autofield bids.author:haxbi~
"""
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
doc="""specify the dataset to perform the query operation on. If
no dataset is given, an attempt is made to identify the dataset
based on the current working directory and/or the `path` given""",
constraints=EnsureDataset() | EnsureNone()),
query=Parameter(
args=("query", ),
metavar='QUERY',
nargs="*",
doc="""query string, supported syntax and features depends on the
selected search mode (see documentation)"""),
force_reindex=Parameter(
args=("--reindex", ),
dest='force_reindex',
action='store_true',
doc="""force rebuilding the search index, even if no change in the
dataset's state has been detected, for example, when the index
documenttype configuration has changed."""),
max_nresults=Parameter(
args=("--max-nresults", ),
doc="""maxmimum number of search results to report. Setting this
to 0 will report all search matches. Depending on the mode this
can search substantially slower. If not specified, a
mode-specific default setting will be used.""",
constraints=EnsureInt() | EnsureNone()),
mode=Parameter(
args=("--mode", ),
choices=('egrep', 'textblob', 'autofield'),
doc="""Mode of search index structure and content. See section
SEARCH MODES for details.
"""),
full_record=Parameter(
args=("--full-record", '-f'),
action='store_true',
doc="""If set, return the full metadata record for each search hit.
Depending on the search mode this might require additional queries.
By default, only data that is available to the respective search modes
is returned. This always includes essential information, such as the
path and the type."""),
show_keys=Parameter(
args=('--show-keys', ),
choices=('name', 'short', 'full'),
default=None,
doc="""if given, a list of known search keys is shown. If 'name' -
only the name is printed one per line. If 'short' or 'full',
statistics (in how many datasets, and how many unique values) are
printed. 'short' truncates the listing of unique values.
No other action is performed (except for reindexing), even if other
arguments are given. Each key is accompanied by a term definition in
parenthesis (TODO). In most cases a definition is given in the form
of a URL. If an ontology definition for a term is known, this URL
can resolve to a webpage that provides a comprehensive definition
of the term. However, for speed reasons term resolution is solely done
on information contained in a local dataset's metadata, and definition
URLs might be outdated or point to no longer existing resources."""
),
show_query=Parameter(
args=('--show-query', ),
action='store_true',
doc="""if given, the formal query that was generated from the given
query string is shown, but not actually executed. This is mostly useful
for debugging purposes."""),
)
@staticmethod
@datasetmethod(name='search')
@eval_results
def __call__(query=None,
dataset=None,
force_reindex=False,
max_nresults=None,
mode=None,
full_record=False,
show_keys=None,
show_query=False):
try:
ds = require_dataset(dataset,
check_installed=True,
purpose='dataset search')
if ds.id is None:
raise NoDatasetArgumentFound(
"This does not seem to be a dataset (no DataLad dataset ID "
"found). 'datalad create --force %s' can initialize "
"this repository as a DataLad dataset" % ds.path)
except NoDatasetArgumentFound:
for r in _search_from_virgin_install(dataset, query):
yield r
return
if mode is None:
# let's get inspired by what the dataset/user think is
# default
mode = ds.config.obtain('datalad.search.default-mode')
if mode == 'egrep':
searcher = _EGrepSearch
elif mode == 'egrepcs':
searcher = _EGrepCSSearch
elif mode == 'textblob':
searcher = _BlobSearch
elif mode == 'autofield':
searcher = _AutofieldSearch
else:
raise ValueError('unknown search mode "{}"'.format(mode))
searcher = searcher(ds, force_reindex=force_reindex)
if show_keys:
searcher.show_keys(show_keys)
return
if not query:
return
if show_query:
print(repr(searcher.get_query(query)))
return
for r in searcher(query,
max_nresults=max_nresults,
full_record=full_record):
yield r
class Get(Interface):
"""Get any dataset content (files/directories/subdatasets).
This command only operates on dataset content. To obtain a new independent
dataset from some source use the `install` command.
By default this command operates recursively within a dataset, but not
across potential subdatasets, i.e. if a directory is provided, all files in
the directory are obtained. Recursion into subdatasets is supported too. If
enabled, relevant subdatasets are detected and installed in order to
fulfill a request.
Known data locations for each requested file are evaluated and data are
obtained from some available location (according to git-annex configuration
and possibly assigned remote priorities), unless a specific source is
specified.
.. note::
Power-user info: This command uses :command:`git annex get` to fulfill
file handles.
"""
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
metavar="PATH",
doc="""specify the dataset to perform the add operation on, in
which case `path` arguments are interpreted as being relative
to this dataset. If no dataset is given, an attempt is made to
identify a dataset for each input `path`""",
constraints=EnsureDataset() | EnsureNone()),
path=Parameter(
args=("path", ),
metavar="PATH",
doc="""path/name of the requested dataset component. The component
must already be known to a dataset. To add new components to a
dataset use the `add` command""",
nargs="*",
constraints=EnsureStr() | EnsureNone()),
source=Parameter(
args=(
"-s",
"--source",
),
metavar="LABEL",
doc="""label of the data source to be used to fulfill requests.
This can be the name of a dataset :term:`sibling` or another known
source""",
constraints=EnsureStr() | EnsureNone()),
recursive=recursion_flag,
recursion_limit=Parameter(
args=("--recursion-limit", ),
metavar="LEVELS",
constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(),
doc="""limit recursion into subdataset to the given number of levels.
Alternatively, 'existing' will limit recursion to subdatasets that already
existed on the filesystem at the start of processing, and prevent new
subdatasets from being obtained recursively."""),
get_data=Parameter(
args=(
"-n",
"--no-data",
),
dest='get_data',
action='store_false',
doc=
"""whether to obtain data for all file handles. If disabled, `get`
operations are limited to dataset handles.[CMD: This option prevents data
for file handles from being obtained CMD]"""),
description=location_description,
reckless=reckless_opt,
# git_opts=git_opts,
# annex_opts=annex_opts,
# annex_get_opts=annex_get_opts,
jobs=jobs_opt,
verbose=verbose)
# Note: May be use 'git annex find --not --in here' to have a list of all
# files to actually get and give kind of a progress in terms of number
# files processed ...
@staticmethod
@datasetmethod(name='get')
@eval_results
def __call__(
path=None,
source=None,
dataset=None,
recursive=False,
recursion_limit=None,
get_data=True,
description=None,
reckless=False,
#git_opts=None,
#annex_opts=None,
#annex_get_opts=None,
jobs='auto',
verbose=False,
):
# IMPLEMENTATION CONCEPT:
#
# 1. Sort the world into existing handles and the rest
# 2. Try locate missing handles (obtain subdatasets along the way)
# 3. Expand into subdatasets with recursion enables (potentially
# obtain even more subdatasets
# 4. Shoot info of which handles to get in each subdataset to,
# git-annex, once at the very end
refds_path = Interface.get_refds_path(dataset)
if not (dataset or path):
raise InsufficientArgumentsError(
"Neither dataset nor target path(s) provided")
if dataset and not path:
# act on the whole dataset if nothing else was specified
path = refds_path
# remember which results we already reported, to avoid duplicates
yielded_ds = []
to_get = []
unavailable_paths = []
for ap in AnnotatePaths.__call__(
path=path,
dataset=refds_path,
recursive=recursive,
recursion_limit=recursion_limit,
action='get',
# NOTE: Do not act upon unavailable paths yet! Done below after
# testing which ones could be obtained
unavailable_path_status='',
nondataset_path_status='impossible',
return_type='generator',
on_failure='ignore'):
if ap.get('status', None):
# we know what to report already
yield ap
continue
if ap.get('state', None) == 'absent' and ap.get(
'raw_input', False):
# if this wasn't found, but directly requested, queue for further
# exploration
unavailable_paths.append(ap)
continue
if ap.get('type', None) == 'dataset' and \
GitRepo.is_valid_repo(ap['path']) and \
not ap['path'] == refds_path:
# do not report what hasn't arived yet
# also do not report the base dataset that is already
# present -- no surprise
yield dict(ap,
status='notneeded',
logger=lgr,
message='already installed')
yielded_ds.append(ap['path'])
ap['process_content'] = get_data
to_get.append(ap)
# explore the unknown
for ap in sorted(unavailable_paths, key=lambda x: x['path']):
lgr.debug("Investigate yet unavailable path %s", ap)
# how close can we get?
dspath = ap.get('parentds', get_dataset_root(ap['path']))
if dspath is None:
# nothing we can do for this path
continue
lgr.debug("Found containing dataset %s for path %s", dspath,
ap['path'])
ds = Dataset(dspath)
# now actually obtain whatever is necessary to get to this path
containing_ds = [dspath]
for res in _install_necessary_subdatasets(ds,
ap['path'],
reckless,
refds_path,
description=description):
# yield immediately so errors could be acted upon outside, before
# we continue
if not (res['type'] == 'dataset'
and res['path'] in yielded_ds):
# unless we reported on this dataset before
if res['type'] == 'dataset':
# make a record, recursive below might now want to report
# a 'notneeded'
yielded_ds.append(res['path'])
yield res
# update to the current innermost dataset
containing_ds.append(res['path'])
if len(containing_ds) < 2:
# no subdataset was installed, hence if the path was unavailable
# before it still is, no need to bother git annex
ap.update(status='impossible', message='path does not exist')
yield ap
continue
# important to only do the next for the innermost subdataset
# as the `recursive` logic below relies on that!
# set the correct parent, for a dataset this would be the second-last
# reported subdataset
ap.update(parentds=containing_ds[-1])
if containing_ds[-1] == ap['path']:
# the path actually refers to the last installed dataset
ap.update(parentds=containing_ds[-2],
process_content=get_data,
type='dataset')
to_get.append(ap)
# results of recursive installation of yet undiscovered datasets
rec_get = []
if recursive and not recursion_limit == 'existing':
# obtain any subdatasets underneath the paths given inside the
# subdatasets that we know already exist
# unless we do not want recursion into not-yet-installed datasets
for ap in sorted(to_get, key=lambda x: x['path']):
if ap['type'] not in ('dataset', 'directory') or not ap.get(
'raw_input', False):
# a non-directory cannot have content underneath
# also we do NOT want to recurse into anything that was specifically
# requested, to avoid duplication
continue
subds = Dataset(ap['path'] if ap['type'] ==
'dataset' else ap['parentds'])
lgr.info("Installing %s%s recursively", subds,
(" underneath %s" %
ap['path'] if subds.path != ap['path'] else ""))
for res in _recursive_install_subds_underneath(
subds,
# `ap['path']` was explicitly given as input
# we count recursions from the input, hence we
# can start with the full number
recursion_limit,
reckless,
start=ap['path'],
refds_path=refds_path,
description=description):
# yield immediately so errors could be acted upon
# outside, before we continue
if not (res['type'] == 'dataset'
and res['path'] in yielded_ds):
# unless we reported on this dataset before
if res['type'] == 'dataset':
# make a record
yielded_ds.append(res['path'])
yield res
if not (res['status'] == 'ok'
and res['type'] == 'dataset'):
# not a dataset that was just installed, we just reported it
# upstairs, and can ignore it from now on
continue
# paranoia, so popular these days...
assert GitRepo.is_valid_repo(res['path'])
# keep a copy of the install record for `get` later on
get_ap = {
k: v
for k, v in res.items() if not k == 'status'
}
get_ap['process_content'] = get_data
rec_get.append(get_ap)
if not get_data:
# done already
return
# merge the two AP lists
to_get.extend(rec_get)
# sort into datasets
content_by_ds, ds_props, completed, nondataset_paths = \
annotated2content_by_ds(
to_get,
refds_path=refds_path)
assert (not completed)
# hand over to git-annex, get files content,
# report files in git as 'notneeded' to get
for ds_path in sorted(content_by_ds.keys()):
ds = Dataset(ds_path)
# grab content, ignore subdataset entries
content = [
ap['path'] for ap in content_by_ds[ds_path]
if ap.get('type', None) != 'dataset' or ap['path'] == ds.path
]
if not content:
# cut this short should there be nothing
continue
# needs to be an annex to get content
if not isinstance(ds.repo, AnnexRepo):
for r in results_from_paths(
content,
status='notneeded',
message="no dataset annex, content already present",
action='get',
logger=lgr,
refds=refds_path):
yield r
continue
respath_by_status = {}
for res in ds.repo.get(content,
options=['--from=%s' %
source] if source else [],
jobs=jobs):
res = annexjson2result(res,
ds,
type='file',
logger=lgr,
refds=refds_path)
success = success_status_map[res['status']]
# TODO: in case of some failed commands (e.g. get) there might
# be no path in the record. yoh has only vague idea of logic
# here so just checks for having 'path', but according to
# results_from_annex_noinfo, then it would be assumed that
# `content` was acquired successfully, which is not the case
if 'path' in res:
respath_by_status[success] = \
respath_by_status.get(success, []) + [res['path']]
yield res
for r in results_from_annex_noinfo(
ds,
content,
respath_by_status,
dir_fail_msg='could not get some content in %s %s',
noinfo_dir_msg='nothing to get from %s',
noinfo_file_msg='already present',
action='get',
logger=lgr,
refds=refds_path):
yield r
@staticmethod
def custom_result_summary_renderer(res):
from datalad.ui import ui
from os import linesep
if not len(res):
ui.message("Got nothing new")
return
nfiles = count_results(res, type='file')
nsuccess_file = count_results(res, type='file', status='ok')
nfailure = nfiles - nsuccess_file
msg = "Tried to get %d %s that had no content yet." % (
nfiles, single_or_plural("file", "files", nfiles))
if nsuccess_file:
msg += " Successfully obtained %d. " % nsuccess_file
if nfailure:
msg += " %d (failed)." % (nfailure, )
ui.message(msg)
# if just a few or less than initially explicitly requested
if len(res) < 10:
msg = linesep.join([
"{path}{type} ... {suc}".format(
suc=item.get('status'),
path=item.get('path'),
type=' [{}]'.format(item['type'])
if 'type' in item else '') for item in res
])
ui.message(msg)
recursion_flag = Parameter(
args=(
"-r",
"--recursive",
),
action="store_true",
doc="""if set, recurse into potential subdatasets""")
recursion_limit = Parameter(
args=(
"-R",
"--recursion-limit",
),
metavar="LEVELS",
constraints=EnsureInt() | EnsureNone(),
doc="""limit recursion into subdatasets to the given number of levels""")
contains = Parameter(args=('--contains', ),
metavar='PATH',
action='append',
doc="""limit to the subdatasets containing the
given path. If a root path of a subdataset is given, the last
considered dataset will be the subdataset itself.[CMD: This
option can be given multiple times CMD][PY: Can be a list with
multiple paths PY], in which case datasets that
contain any of the given paths will be considered.""",
constraints=EnsureStr() | EnsureNone())
fulfilled = Parameter(args=("--fulfilled", ),
doc="""DEPRECATED: use [CMD: --state CMD][PY: `state` PY]
class ExportToFigshare(Interface):
"""Export the content of a dataset as a ZIP archive to figshare
Very quick and dirty approach. Ideally figshare should be supported as
a proper git annex special remote. Unfortunately, figshare does not support
having directories, and can store only a flat list of files. That makes
it impossible for any sensible publishing of complete datasets.
The only workaround is to publish dataset as a zip-ball, where the entire
content is wrapped into a .zip archive for which figshare would provide a
navigator.
"""
from datalad.support.param import Parameter
from datalad.distribution.dataset import datasetmethod
from datalad.interface.utils import eval_results
from datalad.distribution.dataset import EnsureDataset
from datalad.support.constraints import EnsureNone, EnsureInt, EnsureStr
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
doc=""""specify the dataset to export. If no dataset is given, an
attempt is made to identify the dataset based on the current
working directory.""",
constraints=EnsureDataset() | EnsureNone()),
filename=Parameter(
args=("filename",),
metavar="PATH",
nargs='?',
doc="""File name of the generated ZIP archive. If no file name is
given the archive will be generated in the top directory
of the dataset and will be named: datalad_<dataset_uuid>.zip.""",
constraints=EnsureStr() | EnsureNone()),
no_annex=Parameter(
args=("--no-annex",),
action="store_true",
doc="""By default the generated .zip file would be added to annex,
and all files would get registered in git-annex to be available
from such a tarball. Also upon upload we will register for that
archive to be a possible source for it in annex. Setting this flag
disables this behavior."""),
missing_content=Parameter(
args=("--missing-content",),
metavar="error|continue|ignore",
doc="""By default, any discovered file with missing content will
result in an error and the plugin is aborted. Setting this to
'continue' will issue warnings instead of failing on error. The
value 'ignore' will only inform about problem at the 'debug' log
level. The latter two can be helpful when generating a TAR archive
from a dataset where some file content is not available
locally.""",
constraints=EnsureStr()),
# article_id=Parameter(
# args=("--project-id",),
# metavar="ID",
# doc="""If given, article (if article_id is not provided) will be
# created in that project.""",
# constraints=EnsureInt() | EnsureNone()),
article_id=Parameter(
args=("--article-id",),
metavar="ID",
doc="""Which article to publish to.""",
constraints=EnsureInt() | EnsureNone()),
)
@staticmethod
@datasetmethod(name='export_to_figshare')
@eval_results
def __call__(dataset, filename=None, missing_content='error', no_annex=False,
# TODO: support working with projects and articles within them
# project_id=None,
article_id=None):
import os
import logging
lgr = logging.getLogger('datalad.plugin.export_to_figshare')
from datalad.ui import ui
from datalad.api import add_archive_content
from datalad.api import export_archive
from datalad.distribution.dataset import require_dataset
from datalad.support.annexrepo import AnnexRepo
dataset = require_dataset(dataset, check_installed=True,
purpose='export to figshare')
if not isinstance(dataset.repo, AnnexRepo):
raise ValueError(
"%s is not an annex repo, so annexification could be done"
% dataset
)
if dataset.repo.is_dirty():
raise RuntimeError(
"Paranoid authors of DataLad refuse to proceed in a dirty repository"
)
if filename is None:
filename = dataset.path
lgr.info(
"Exporting current tree as an archive under %s since figshare "
"does not support directories",
filename
)
archive_out = next(
export_archive(
dataset,
filename=filename,
archivetype='zip',
missing_content=missing_content,
return_type="generator"
)
)
assert archive_out['status'] == 'ok'
fname = archive_out['path']
lgr.info("Uploading %s to figshare", fname)
figshare = FigshareRESTLaison()
if not article_id:
# TODO: ask if it should be an article within a project
if ui.is_interactive:
# or should we just upload to a new article?
if ui.yesno(
"Would you like to create a new article to upload to? "
"If not - we will list existing articles",
title="Article"
):
article = figshare.create_article(
title=os.path.basename(dataset.path)
)
lgr.info(
"Created a new (private) article %(id)s at %(url_private_html)s. "
"Please visit it, enter additional meta-data and make public",
article
)
article_id = article['id']
else:
article_id = int(ui.question(
"Which of the articles should we upload to.",
choices=list(map(str, figshare.get_article_ids()))
))
if not article_id:
raise ValueError("We need an article to upload to.")
file_info = figshare.upload_file(
fname,
files_url='account/articles/%s/files' % article_id
)
if no_annex:
lgr.info("Removing generated tarball")
unlink(fname)
else:
# I will leave all the complaining etc to the dataset add if path
# is outside etc
lgr.info("'Registering' %s within annex", fname)
repo = dataset.repo
repo.add(fname, git=False)
key = repo.get_file_key(fname)
lgr.info("Adding URL %(download_url)s for it", file_info)
repo._annex_custom_command([],
[
"git", "annex", "registerurl", '-c', 'annex.alwayscommit=false',
key, file_info['download_url']
]
)
lgr.info("Registering links back for the content of the archive")
add_archive_content(
fname,
annex=dataset.repo,
delete_after=True, # just remove extracted into a temp dir
allow_dirty=True, # since we have a tarball
commit=False # we do not want to commit anything we have done here
)
lgr.info("Removing generated and now registered in annex archive")
repo.drop(key, key=True, options=['--force'])
repo.remove(fname, force=True) # remove the tarball
# if annex in {'delete'}:
# dataset.repo.remove(fname)
# else:
# # kinda makes little sense I guess.
# # Made more sense if export_archive could export an arbitrary treeish
# # so we could create a branch where to dump and export to figshare
# # (kinda closer to my idea)
# dataset.save(fname, message="Added the entire dataset into a zip file")
# TODO: add to downloader knowledge about figshare token so it could download-url
# those zipballs before they go public
yield dict(
status='ok',
# TODO: add article url (which needs to be queried if only ID is known
message="Published archive {}".format(
file_info['download_url']),
file_info=file_info,
path=dataset,
action='export_to_figshare',
logger=lgr
)
'destination':
'local',
'type':
EnsureBool(),
'default':
False,
},
'datalad.fake-dates-start': {
'ui': ('question', {
'title':
'Initial fake date',
'text':
'When faking dates and there are no commits in any local branches, generate the date by adding one second to this value (Unix epoch time). The value must be positive.'
}),
'type':
EnsureInt(),
'default':
1112911993,
},
'datalad.github.token-note': {
'ui': ('question', {
'title': 'Github token note',
'text': 'Description for a Personal access token to generate.'
}),
'default':
'DataLad',
},
'datalad.tests.nonetwork': {
'ui': ('yesno', {
'title':
'Skips network tests completely if this flag is set Examples include test for s3, git_repositories, openfmri etc'
class Get(Interface):
"""Get any dataset content (files/directories/subdatasets).
This command only operates on dataset content. To obtain a new independent
dataset from some source use the `install` command.
By default this command operates recursively within a dataset, but not
across potential subdatasets, i.e. if a directory is provided, all files in
the directory are obtained. Recursion into subdatasets is supported too. If
enabled, relevant subdatasets are detected and installed in order to
fulfill a request.
Known data locations for each requested file are evaluated and data are
obtained from some available location (according to git-annex configuration
and possibly assigned remote priorities), unless a specific source is
specified.
.. note::
Power-user info: This command uses :command:`git annex get` to fulfill
file handles.
"""
_examples_ = [
dict(text="Get a single file",
code_py="get('path/to/file')",
code_cmd="datalad get <path/to/file>"),
dict(text="Get contents of a directory",
code_py="get('path/to/dir/')",
code_cmd="datalad get <path/to/dir/>"),
dict(
text="Get all contents of the current dataset and its subdatasets",
code_py="get(dataset='.', recursive=True)",
code_cmd="datalad get . -r"),
dict(
text="Get (clone) a registered subdataset, but don't retrieve data",
code_py="get('path/to/subds', get_data=False)",
code_cmd="datalad get -n <path/to/subds>"),
]
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
metavar="PATH",
doc="""specify the dataset to perform the add operation on, in
which case `path` arguments are interpreted as being relative
to this dataset. If no dataset is given, an attempt is made to
identify a dataset for each input `path`""",
constraints=EnsureDataset() | EnsureNone()),
path=Parameter(
args=("path", ),
metavar="PATH",
doc="""path/name of the requested dataset component. The component
must already be known to a dataset. To add new components to a
dataset use the `add` command""",
nargs="*",
constraints=EnsureStr() | EnsureNone()),
source=Parameter(
args=(
"-s",
"--source",
),
metavar="LABEL",
doc="""label of the data source to be used to fulfill requests.
This can be the name of a dataset :term:`sibling` or another known
source""",
constraints=EnsureStr() | EnsureNone()),
recursive=recursion_flag,
recursion_limit=Parameter(
args=(
"-R",
"--recursion-limit",
),
metavar="LEVELS",
constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(),
doc="""limit recursion into subdataset to the given number of levels.
Alternatively, 'existing' will limit recursion to subdatasets that already
existed on the filesystem at the start of processing, and prevent new
subdatasets from being obtained recursively."""),
get_data=Parameter(
args=(
"-n",
"--no-data",
),
dest='get_data',
action='store_false',
doc=
"""whether to obtain data for all file handles. If disabled, `get`
operations are limited to dataset handles.[CMD: This option prevents data
for file handles from being obtained CMD]"""),
description=location_description,
reckless=reckless_opt,
jobs=jobs_opt)
@staticmethod
@datasetmethod(name='get')
@eval_results
def __call__(
path=None,
source=None,
dataset=None,
recursive=False,
recursion_limit=None,
get_data=True,
description=None,
reckless=None,
jobs='auto',
):
refds_path = Interface.get_refds_path(dataset)
if not (dataset or path):
raise InsufficientArgumentsError(
"Neither dataset nor target path(s) provided")
if dataset and not path:
# act on the whole dataset if nothing else was specified
path = refds_path
# we have to have a single dataset to operate on
refds = require_dataset(dataset,
check_installed=True,
purpose='get content')
content_by_ds = {}
# use subdatasets() to discover any relevant content that is not
# already present in the root dataset (refds)
for sdsres in Subdatasets.__call__(
contains=path,
# maintain path argument semantics and pass in dataset arg
# as is
dataset=dataset,
# always come from the top to get sensible generator behavior
bottomup=False,
# when paths are given, they will constrain the recursion
# automatically, and we need to enable recursion so we can
# location path in subdatasets several levels down
recursive=True if path else recursive,
recursion_limit=None if path else recursion_limit,
return_type='generator',
on_failure='ignore'):
if sdsres.get('type', None) != 'dataset':
# if it is not about a 'dataset' it is likely content in
# the root dataset
if sdsres.get('status', None) == 'impossible' and \
sdsres.get('message', None) == \
'path not contained in any matching subdataset':
target_path = Path(sdsres['path'])
if refds.pathobj != target_path and \
refds.pathobj not in target_path.parents:
yield dict(
action='get',
path=str(target_path),
status='error',
message=('path not associated with dataset %s',
refds),
)
continue
# check if we need to obtain anything underneath this path
# the subdataset() call above will only look _until_ it
# hits the targetpath
for res in _install_targetpath(
refds,
Path(sdsres['path']),
recursive,
recursion_limit,
reckless,
refds_path,
description,
jobs=jobs,
):
# fish out the datasets that 'contains' a targetpath
# and store them for later
if res.get('status', None) in ('ok', 'notneeded') and \
'contains' in res:
dsrec = content_by_ds.get(res['path'], set())
dsrec.update(res['contains'])
content_by_ds[res['path']] = dsrec
if res.get('status', None) != 'notneeded':
# all those messages on not having installed anything
# are a bit pointless
# "notneeded" for annex get comes below
yield res
else:
# dunno what this is, send upstairs
yield sdsres
# must continue for both conditional branches above
# the rest is about stuff in real subdatasets
continue
# instance of the closest existing dataset for this result
ds = Dataset(sdsres['parentds'] if sdsres.get('state', None) ==
'absent' else sdsres['path'])
assert 'contains' in sdsres
# explore the unknown
for target_path in sdsres.get('contains', []):
# essentially the same as done above for paths in the root
# dataset, but here we are starting from the closest
# discovered subdataset
for res in _install_targetpath(
ds,
Path(target_path),
recursive,
recursion_limit,
reckless,
refds_path,
description,
jobs=jobs,
):
known_ds = res['path'] in content_by_ds
if res.get('status', None) in ('ok', 'notneeded') and \
'contains' in res:
dsrec = content_by_ds.get(res['path'], set())
dsrec.update(res['contains'])
content_by_ds[res['path']] = dsrec
# prevent double-reporting of datasets that have been
# installed by explorative installation to get to target
# paths, prior in this loop
if res.get('status', None) != 'notneeded' or not known_ds:
yield res
if not get_data:
# done already
return
# and now annex-get, this could all be done in parallel now
for ds, content in content_by_ds.items():
for res in _get_targetpaths(Dataset(ds), content, refds.path,
source, jobs):
if res['path'] not in content_by_ds:
# we had reports on datasets and subdatasets already
# before the annex stage
yield res
class Search(Interface):
"""Search a dataset's metadata.
Search capabilities depend on the amount and nature of metadata available
in a dataset. This can include metadata about a dataset as a whole, or
metadata on dataset content (e.g. one or more files). One dataset can also
contain metadata from multiple subdatasets (see the 'aggregate-metadata'
command), in which case a search can discover any dataset or any file in
of these datasets.
A search index is automatically built from the available metadata of any
dataset or file, and a schema for this index is generated dynamically, too.
Consequently, the search index will be tailored to data provided in a
particular collection of datasets.
Metadata fields (and possibly also values) are typically defined terms
from a controlled vocabulary. Term definitions are accessible via the
--show-keys flag.
DataLad's search is built on the Python package 'Whoosh', which provides
a powerful query language. Links to a description of the language and
particular feature can be found below.
Here are a few examples. Basic search::
% datalad search searchterm
Search for a file::
% datalad search searchterm type:file
Show definitions of search keys/fields::
% datalad search --show-keys
@id (unique identifier of an entity)
dcterms:rights (http://purl.org/dc/terms/rights)
duration(s) {'unit (http://purl.obolibrary.org/obo/UO_0000000)': 'http://purl.obolibrary.org/obo/UO_0000010', '@id': 'https://www.w3.org/TR/owl-time/#Duration'}
name (http://schema.org/name)
...
*Performance considerations*
For dataset collections with many files (100k+) generating a comprehensive
search index comprised of documents for datasets and individual files can
take a considerable amount of time. If this becomes an issue, search index
generation can be limited to a particular type of document (see the
'metadata --reporton' option for possible values). The configuration
setting 'datalad.metadata.searchindex-documenttype' will be queried on
search index generation. It is recommended to place an appropriate
configuration into a dataset's configuration file (.datalad/config)::
[datalad "metadata"]
searchindex-documenttype = datasets
.. seealso::
- Description of the Whoosh query language:
http://whoosh.readthedocs.io/en/latest/querylang.html)
- Description of a number of query language customizations that are
enabled in DataLad, such as, querying multiple fields by default and
fuzzy term matching:
http://whoosh.readthedocs.io/en/latest/parsing.html#common-customizations
"""
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
doc="""specify the dataset to perform the query operation on. If
no dataset is given, an attempt is made to identify the dataset
based on the current working directory and/or the `path` given""",
constraints=EnsureDataset() | EnsureNone()),
query=Parameter(
args=("query", ),
metavar='QUERY',
nargs="*",
doc="""search query using the Whoosh query language (see link to
detailed description above). For simple queries, any number of search
terms can be given as a list[CMD: (space-separated) CMD], and the
query will return all hits that match all terms (AND) in any combination
of fields (OR)."""),
force_reindex=Parameter(
args=("--reindex", ),
dest='force_reindex',
action='store_true',
doc="""force rebuilding the search index, even if no change in the
dataset's state has been detected. This is mostly useful for
developing new metadata support extensions."""),
max_nresults=Parameter(
args=("--max-nresults", ),
doc="""maxmimum number of search results to report. Setting this
to 0 will report all search matches, and make searching substantially
slower on large metadata sets.""",
constraints=EnsureInt()),
show_keys=Parameter(
args=('--show-keys', ),
action='store_true',
doc="""if given, a list of known search keys is shown (one per line).
No other action is performed (except for reindexing), even if other
arguments are given. Each key is accompanied by a term definition in
parenthesis. In most cases a definition is given in the form
of a URL. If an ontology definition for a term is known, this URL
can resolve to a webpage that provides a comprehensive definition
of the term. However, for speed reasons term resolution is solely done
on information contained in a local dataset's metadata, and definition
URLs might be outdated or point to no longer existing resources."""
),
show_query=Parameter(
args=('--show-query', ),
action='store_true',
doc="""if given, the formal query that was generated from the given
query string is shown, but not actually executed. This is mostly useful
for debugging purposes."""),
)
@staticmethod
@datasetmethod(name='search')
@eval_results
def __call__(query=None,
dataset=None,
force_reindex=False,
max_nresults=20,
show_keys=False,
show_query=False):
from whoosh import qparser as qparse
try:
ds = require_dataset(dataset,
check_installed=True,
purpose='dataset search')
if ds.id is None:
raise NoDatasetArgumentFound(
"This does not seem to be a dataset (no DataLad dataset ID "
"found). 'datalad create --force %s' can initialize "
"this repository as a DataLad dataset" % ds.path)
except NoDatasetArgumentFound:
for r in _search_from_virgin_install(dataset, query):
yield r
return
# where does the bunny have the eggs?
index_dir = opj(ds.path, get_git_dir(ds.path), 'datalad',
'search_index')
idx_obj = _get_search_index(index_dir, ds, force_reindex)
if show_keys:
definitions_fname = opj(index_dir,
'datalad_term_definitions.json.gz')
try:
defs = jsonload(gzopen(definitions_fname))
except Exception as e:
lgr.warning(
'No term definitions found alongside search index: %s',
exc_str(e))
defs = {}
for k in idx_obj.schema.names():
print('{}{}'.format(
k,
' {}'.format(defs[k] if isinstance(defs[k], dict) else
'({})'.format(defs[k])) if k in defs else ''))
return
if not query:
return
with idx_obj.searcher() as searcher:
# parse the query string, default whoosh parser ATM, could be
# tailored with plugins
parser = qparse.MultifieldParser(idx_obj.schema.names(),
idx_obj.schema)
# XXX: plugin is broken in Debian's whoosh 2.7.0-2, but already fixed
# upstream
parser.add_plugin(qparse.FuzzyTermPlugin())
parser.add_plugin(qparse.GtLtPlugin())
# replace field defintion to allow for colons to be part of a field's name:
parser.replace_plugin(
qparse.FieldsPlugin(expr=r"(?P<text>[()<>:\w]+|[*]):"))
# for convenience we accept any number of args-words from the
# shell and put them together to a single string here
querystr = ' '.join(assure_list(query))
# this gives a formal whoosh query
wquery = parser.parse(querystr)
if show_query:
print(wquery)
return
# perform the actual search
hits = searcher.search(
wquery,
terms=True,
limit=max_nresults if max_nresults > 0 else None)
# cheap way to get an approximate number of hits, without an expensive
# scoring of all items
# disabled: unreliable estimate, often confusing
#nhits = hits.estimated_min_length()
# report query stats
topstr = '{} top {}'.format(
max_nresults, single_or_plural('match', 'matches',
max_nresults))
lgr.info('Query completed in {} sec.{}'.format(
hits.runtime, ' Reporting {}.'.format((
'up to ' + topstr) if max_nresults > 0 else 'all matches')
if not hits.is_empty() else ' No matches.'))
if not hits:
return
nhits = 0
for hit in hits:
res = dict(
action='search',
status='ok',
logger=lgr,
refds=ds.path,
# normpath to avoid trailing dot
path=normpath(opj(ds.path, hit['path'])),
query_matched={
assure_unicode(k): assure_unicode(v) if isinstance(
v, unicode_srctypes) else v
for k, v in hit.matched_terms()
},
metadata={
k: v
for k, v in hit.fields().items()
if k not in ('path', 'parentds')
})
if 'parentds' in hit:
res['parentds'] = normpath(opj(ds.path, hit['parentds']))
yield res
nhits += 1
if max_nresults and nhits == max_nresults:
lgr.info("Reached the limit of {}, there could be more which "
"were not reported.".format(topstr))
'ui': ('question', {
'title': 'This flag is used by the datalad extract_tb function which extracts and formats stack-traces. It caps the number of lines to DATALAD_EXC_STR_TBLIMIT of pre-processed entries from traceback.'}),
},
'datalad.fake-dates': {
'ui': ('yesno', {
'title': 'Fake (anonymize) dates',
'text': 'Should the dates in the logs be faked?'}),
'destination': 'local',
'type': EnsureBool(),
'default': False,
},
'datalad.fake-dates-start': {
'ui': ('question', {
'title': 'Initial fake date',
'text': 'When faking dates and there are no commits in any local branches, generate the date by adding one second to this value (Unix epoch time). The value must be positive.'}),
'type': EnsureInt(),
'default': 1112911993,
},
'datalad.tests.nonetwork': {
'ui': ('yesno', {
'title': 'Skips network tests completely if this flag is set Examples include test for s3, git_repositories, openfmri etc'}),
'type': EnsureBool(),
},
'datalad.tests.nonlo': {
'ui': ('question', {
'title': 'Specifies network interfaces to bring down/up for testing. Currently used by travis.'}),
},
'datalad.tests.noteardown': {
'ui': ('yesno', {
'title': 'Does not execute teardown_package which cleans up temp files and directories created by tests if this flag is set'}),
'type': EnsureBool(),
class Get(Interface):
"""Get any dataset content (files/directories/subdatasets).
This command only operates on dataset content. To obtain a new independent
dataset from some source use the `install` command.
By default this command operates recursively within a dataset, but not
across potential subdatasets, i.e. if a directory is provided, all files in
the directory are obtained. Recursion into subdatasets is supported too. If
enabled, relevant subdatasets are detected and installed in order to
fulfill a request.
Known data locations for each requested file are evaluated and data are
obtained from some available location (according to git-annex configuration
and possibly assigned remote priorities), unless a specific source is
specified.
.. note::
Power-user info: This command uses :command:`git annex get` to fulfill
file handles.
"""
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
metavar="PATH",
doc="""specify the dataset to perform the add operation on, in
which case `path` arguments are interpreted as being relative
to this dataset. If no dataset is given, an attempt is made to
identify a dataset for each input `path`""",
constraints=EnsureDataset() | EnsureNone()),
path=Parameter(
args=("path", ),
metavar="PATH",
doc="""path/name of the requested dataset component. The component
must already be known to a dataset. To add new components to a
dataset use the `add` command""",
nargs="*",
constraints=EnsureStr() | EnsureNone()),
source=Parameter(
args=(
"-s",
"--source",
),
metavar="LABEL",
doc="""label of the data source to be used to fulfill requests.
This can be the name of a dataset :term:`sibling` or another known
source""",
constraints=EnsureStr() | EnsureNone()),
recursive=recursion_flag,
recursion_limit=Parameter(
args=("--recursion-limit", ),
metavar="LEVELS",
constraints=EnsureInt() | EnsureChoice('existing') | EnsureNone(),
doc="""limit recursion into subdataset to the given number of levels.
Alternatively, 'existing' will limit recursion to subdatasets that already
existed on the filesystem at the start of processing, and prevent new
subdatasets from being obtained recursively."""),
get_data=Parameter(
args=(
"-n",
"--no-data",
),
dest='get_data',
action='store_false',
doc=
"""whether to obtain data for all file handles. If disabled, `get`
operations are limited to dataset handles.[CMD: This option prevents data
for file handles from being obtained CMD]"""),
reckless=reckless_opt,
git_opts=git_opts,
annex_opts=annex_opts,
annex_get_opts=annex_get_opts,
jobs=jobs_opt,
verbose=verbose)
# Note: May be use 'git annex find --not --in here' to have a list of all
# files to actually get and give kind of a progress in terms of number
# files processed ...
@staticmethod
@datasetmethod(name='get')
def __call__(
path,
source=None,
dataset=None,
recursive=False,
recursion_limit=None,
get_data=True,
reckless=False,
git_opts=None,
annex_opts=None,
annex_get_opts=None,
jobs=None,
verbose=False,
# internal -- instead of returning 'get'ed items, return final
# content_by_ds, unavailable_paths. To be used by the call from
# Install.__call__ and done so to avoid creating another reusable
# function which would need to duplicate all this heavy list of
# kwargs
_return_datasets=False):
# IMPLEMENTATION CONCEPT:
#
# 1. turn all input paths into absolute paths
# 2. Sort the world into existing handles and the rest
# 3. Try locate missing handles (obtain subdatasets along the way)
# 4. Expand into subdatasets with recursion enables (potentially
# obtain even more subdatasets
# 5. Shoot info of which handles to get in each subdataset to,
# git-annex, once at the very end
# TODO: consider allowing an empty `path` argument, as with other commands,
# to indicate CWD
resolved_paths, dataset_path = get_normalized_path_arguments(
path, dataset, default=None)
if not resolved_paths:
raise InsufficientArgumentsError(
"`get` needs at least one path as argument")
# sort paths into the respective datasets
dir_lookup = {}
content_by_ds, unavailable_paths, nondataset_paths = \
get_paths_by_dataset(resolved_paths,
recursive=recursive,
recursion_limit=recursion_limit,
dir_lookup=dir_lookup)
lgr.debug(
"Found %i existing dataset(s) to get content in "
"and %d unavailable paths", len(content_by_ds),
len(unavailable_paths))
# IMPORTANT NOTE re `content_by_ds`
# each key is a subdataset that we need to get something in
# if the value[0] is the subdataset's path, we want all of it
# if the value[0] == curdir, we just installed it as part of
# resolving file handles and we did not say anything but "give
# me the dataset handle"
# explore the unknown
for path in sorted(unavailable_paths):
# how close can we get?
dspath = GitRepo.get_toppath(path)
if dspath is None:
# nothing we can do for this path
continue
ds = Dataset(dspath)
# must always yield a dataset -- we sorted out the ones outside
# any dataset at the very top
assert ds.is_installed()
# now actually obtain whatever is necessary to get to this path
containing_ds = install_necessary_subdatasets(ds, path, reckless)
if containing_ds.path != ds.path:
lgr.debug(
"Installed %s to fulfill request for content for "
"path %s", containing_ds, path)
# mark resulting dataset as auto-installed
if containing_ds.path == path:
# we had to get the entire dataset, not something within
# mark that it just appeared
content_by_ds[path] = [curdir]
else:
# we need to get content within
content_by_ds[path] = [path]
if recursive and not recursion_limit == 'existing':
# obtain any subdatasets underneath the paths given inside the
# subdatasets that we know already exist
# unless we do not want recursion into not-yet-installed datasets
for subdspath in sorted(content_by_ds.keys()):
for content_path in content_by_ds[subdspath]:
if not isdir(content_path):
# a non-directory cannot have content underneath
continue
subds = Dataset(subdspath)
lgr.info(
"Obtaining %s %s recursively", subds,
("underneath %s" %
content_path if subds.path != content_path else ""))
cbysubds = _recursive_install_subds_underneath(
subds,
# `content_path` was explicitly given as input
# we count recursions from the input, hence we
# can start with the full number
recursion_limit,
reckless,
# protect against magic marker misinterpretation
# only relevant for _get, hence replace here
start=content_path if content_path != curdir else None)
# gets file content for all freshly installed subdatasets
content_by_ds.update(cbysubds)
## we have now done everything we could to obtain whatever subdataset
## to get something on the file system for previously unavailable paths
## check and sort one last
content_by_ds, unavailable_paths, nondataset_paths2 = \
get_paths_by_dataset(
unavailable_paths,
recursive=recursive,
recursion_limit=recursion_limit,
out=content_by_ds,
dir_lookup=dir_lookup)
nondataset_paths.extend(nondataset_paths2)
if nondataset_paths:
lgr.warning("ignored paths that do not belong to any dataset: %s",
nondataset_paths)
if unavailable_paths:
lgr.warning('ignored non-existing paths: %s', unavailable_paths)
# hand over to git-annex
results = list(
chain.from_iterable(
_get(content_by_ds,
refpath=dataset_path,
source=source,
jobs=jobs,
get_data=get_data)))
# ??? should we in _return_datasets case just return both content_by_ds
# and unavailable_paths may be so we provide consistent across runs output
# and then issue outside similar IncompleteResultsError?
if unavailable_paths: # and likely other error flags
if _return_datasets:
results = sorted(
set(content_by_ds).difference(unavailable_paths))
raise IncompleteResultsError(results, failed=unavailable_paths)
else:
return sorted(content_by_ds) if _return_datasets else results
@staticmethod
def result_renderer_cmdline(res, args):
from datalad.ui import ui
from os import linesep
if res is None:
res = []
if not isinstance(res, list):
res = [res]
if not len(res):
ui.message("Got nothing new")
return
# provide summary
nsuccess = sum(
item.get('success', False) if isinstance(item, dict) else True
for item in res)
nfailure = len(res) - nsuccess
msg = "Tried to get %d %s." % (
len(res), single_or_plural("file", "files", len(res)))
if nsuccess:
msg += " Got %d. " % nsuccess
if nfailure:
msg += " Failed to get %d." % (nfailure, )
ui.message(msg)
# if just a few or less than initially explicitly requested
if len(res) < 10 or args.verbose:
msg = linesep.join([
"{path} ... {suc}".format(
suc="ok." if isinstance(item, Dataset)
or item.get('success', False) else "failed. (%s)" %
item.get('note', 'unknown reason'),
path=item.get('file')
if isinstance(item, dict) else item.path) for item in res
])
ui.message(msg)
class Search(Interface):
"""Search a dataset's metadata.
Search capabilities depend on the amount and nature of metadata available
in a dataset. This can include metadata about a dataset as a whole, or
metadata on dataset content (e.g. one or more files). One dataset can also
contain metadata from multiple subdatasets (see the 'aggregate-metadata'
command), in which case a search can discover any dataset or any file in
of these datasets.
*Search modes*
WRITE ME
A search index is automatically built from the available metadata of any
dataset or file, and a schema for this index is generated dynamically, too.
Consequently, the search index will be tailored to data provided in a
particular collection of datasets.
Metadata fields (and possibly also values) are typically defined terms from
a controlled vocabulary. Field names are accessible via the --show-keys
flag.
DataLad's search is built on the Python package 'Whoosh', which provides
a powerful query language. Links to a description of the language and
particular feature can be found below.
Here are a few examples. Basic search::
% datalad search searchterm
Search for a file::
% datalad search searchterm type:file
*Performance considerations*
For dataset collections with many files (100k+) generating a comprehensive
search index comprised of documents for datasets and individual files can
take a considerable amount of time. If this becomes an issue, search index
generation can be limited to a particular type of document (see the
'metadata --reporton' option for possible values). The per-mode configuration
setting 'datalad.search.index-<mode>-documenttype' will be queried on
search index generation. It is recommended to place an appropriate
configuration into a dataset's configuration file (.datalad/config)::
[datalad "search"]
index-default-documenttype = datasets
.. seealso::
- Description of the Whoosh query language:
http://whoosh.readthedocs.io/en/latest/querylang.html)
- Description of a number of query language customizations that are
enabled in DataLad, such as, querying multiple fields by default and
fuzzy term matching:
http://whoosh.readthedocs.io/en/latest/parsing.html#common-customizations
"""
_params_ = dict(
dataset=Parameter(
args=("-d", "--dataset"),
doc="""specify the dataset to perform the query operation on. If
no dataset is given, an attempt is made to identify the dataset
based on the current working directory and/or the `path` given""",
constraints=EnsureDataset() | EnsureNone()),
query=Parameter(
args=("query",),
metavar='QUERY',
nargs="*",
doc="""search query using the Whoosh query language (see link to
detailed description above). For simple queries, any number of search
terms can be given as a list[CMD: (space-separated) CMD], and the
query will return all hits that match all terms (AND) in any combination
of fields (OR)."""),
force_reindex=Parameter(
args=("--reindex",),
dest='force_reindex',
action='store_true',
doc="""force rebuilding the search index, even if no change in the
dataset's state has been detected. This is mostly useful for
developing new metadata support extensions."""),
max_nresults=Parameter(
args=("--max-nresults",),
doc="""maxmimum number of search results to report. Setting this
to 0 will report all search matches, and make searching substantially
slower on large metadata sets.""",
constraints=EnsureInt()),
mode=Parameter(
args=("--mode",),
choices=('egrep', 'textblob', 'autofield'),
doc="""Mode of search index structure and content. See section
SEARCH MODES for details.
"""),
show_keys=Parameter(
args=('--show-keys',),
action='store_true',
doc="""if given, a list of known search keys is shown (one per line).
No other action is performed (except for reindexing), even if other
arguments are given. Each key is accompanied by a term definition in
parenthesis. In most cases a definition is given in the form
of a URL. If an ontology definition for a term is known, this URL
can resolve to a webpage that provides a comprehensive definition
of the term. However, for speed reasons term resolution is solely done
on information contained in a local dataset's metadata, and definition
URLs might be outdated or point to no longer existing resources."""),
show_query=Parameter(
args=('--show-query',),
action='store_true',
doc="""if given, the formal query that was generated from the given
query string is shown, but not actually executed. This is mostly useful
for debugging purposes."""),
)
@staticmethod
@datasetmethod(name='search')
@eval_results
def __call__(query=None,
dataset=None,
force_reindex=False,
max_nresults=20,
mode=None,
show_keys=False,
show_query=False):
try:
ds = require_dataset(dataset, check_installed=True, purpose='dataset search')
if ds.id is None:
raise NoDatasetArgumentFound(
"This does not seem to be a dataset (no DataLad dataset ID "
"found). 'datalad create --force %s' can initialize "
"this repository as a DataLad dataset" % ds.path)
except NoDatasetArgumentFound:
for r in _search_from_virgin_install(dataset, query):
yield r
return
if mode is None:
# let's get inspired by what the dataset/user think is
# default
mode = ds.config.obtain('datalad.search.default-mode')
if mode == 'egrep':
searcher = _EGrepSearch
elif mode == 'textblob':
searcher = _BlobSearch
elif mode == 'autofield':
searcher = _AutofieldSearch
else:
raise ValueError(
'unknown search mode "{}"'.format(mode))
searcher = searcher(ds, force_reindex=force_reindex)
if show_keys:
searcher.show_keys()
return
if not query:
return
if show_query:
print(repr(searcher.get_query(query)))
return
for r in searcher(
query,
max_nresults=max_nresults):
yield r
class HTCResults(Interface):
"""TODO
"""
# make the custom renderer the default one, as the global default renderer
# does not yield meaningful output for this command
result_renderer = 'tailored'
_params_ = dict(
cmd=Parameter(args=("cmd", ),
metavar=("SUBCOMMAND", ),
nargs='?',
doc="""""",
constraints=EnsureChoice('list', 'merge', 'remove')),
dataset=Parameter(
args=("-d", "--dataset"),
doc="""specify the dataset to record the command results in.
An attempt is made to identify the dataset based on the current
working directory. If a dataset is given, the command will be
executed in the root directory of this dataset.""",
constraints=EnsureDataset() | EnsureNone()),
submission=Parameter(args=("submission", ),
nargs='?',
metavar='SUBMISSION',
doc=""""""),
job=Parameter(args=(
"-j",
"--job",
),
metavar='NUMBER',
doc="""""",
constraints=EnsureInt() | EnsureNone()),
all=Parameter(args=("--all", ), action='store_true', doc=""""""),
)
@staticmethod
@datasetmethod(name='htc_results')
@eval_results
def __call__(cmd='list',
dataset=None,
submission=None,
job=None,
all=False):
ds = require_dataset(
dataset,
check_installed=True,
purpose='handling results of remote command executions')
if cmd == 'list':
jw = _list_job
sw = _list_submission
elif cmd == 'merge':
jw = _apply_output
sw = None
elif cmd == 'remove':
if not all and not submission and not job:
raise ValueError(
"use the '--all' flag to remove all results across all "
"submissions")
jw = _remove_dir
sw = _remove_dir
else:
raise ValueError("unknown sub-command '{}'".format(cmd))
for res in _doit(ds, submission, job, jw, sw):
yield res
@staticmethod
def custom_result_renderer(res, **kwargs): # pragma: no cover
from datalad.ui import ui
if not res['status'] == 'ok' or not res['action'].startswith('htc_'):
# logging reported already
return
action = res['action'].split('_')[-1]
ui.message('{action} {sub}{job}{state}{cmd}'.format(
action=ac.color_word(action, kw_color_map.get(action, ac.WHITE))
if action != 'list' else '',
sub=res['submission'],
job=' :{}'.format(res['job']) if 'job' in res else '',
state=' [{}]'.format(
ac.color_word(res['state'],
kw_color_map.get(res['state'], ac.MAGENTA)
) if res.get('state', None) else 'unknown')
if action == 'list' else '',
cmd=': {}'.format(_format_cmd_shorty(res['cmd']))
if 'cmd' in res else '',
))