def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('Dataset information', cmo.out)
assert_in('Configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('Dataset information', cmo.out)
assert_in('Configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('Configuration', cmo.out)
assert_in('Dataset information', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex=", pyperclip.paste()) # but the content is there
def test_check_dates_invalid_date():
skip_if_no_module("dateutil")
with swallow_outputs() as cmo:
assert_raises(IncompleteResultsError,
check_dates, [],
reference_date="not a valid date",
return_type="list")
out = cmo.out
# The error makes it through the standard renderer.
assert_in('"status": "error"', out)
def test_check_dates_invalid_date():
skip_if_no_module("dateutil")
with swallow_outputs() as cmo:
assert_raises(IncompleteResultsError,
check_dates, [],
reference_date="not a valid date",
return_type="list")
out = cmo.out
# The error makes it through the standard renderer.
assert_in('"status": "error"', out)
def test_proxying_lzma_LZMAFile():
skip_if_no_module('lzma')
import lzma
def generate_dat(f):
with LZMAFile(f, "w") as f:
f.write("123".encode('utf-8'))
def verify_dat(f, mode="r"):
with LZMAFile(f, mode) as f:
eq_(f.read().decode('utf-8'), "123")
yield _test_proxying_open, generate_dat, verify_dat
def test_proxying_lzma_LZMAFile():
skip_if_no_module('datalad.support.lzma')
from datalad.support.lzma import lzma
def generate_dat(f):
with LZMAFile(f, "w") as f:
f.write("123".encode('utf-8'))
def verify_dat(f, mode="r"):
with LZMAFile(f, mode) as f:
eq_(f.read().decode('utf-8'), "123")
yield _test_proxying_open, generate_dat, verify_dat
def __test_basic_openfmri_top_pipeline():
skip_if_no_module('scrapy') # e.g. not present under Python3
sink1 = Sink()
sink2 = Sink()
sink_licenses = Sink()
pipeline = [
crawl_url("https://openfmri.org/data-sets"),
a_href_match(
".*/dataset/(?P<dataset_dir>ds0*(?P<dataset>[1-9][0-9]*))$"),
# if we wanted we could instruct to crawl inside
[
crawl_url(),
[ # and collect all URLs under "AWS Link"
css_match('.field-name-field-aws-link a',
xpaths={
'url': '@href',
'url_text': 'text()'
}), sink2
],
[ # and license information
css_match('.field-name-field-license a',
xpaths={
'url': '@href',
'url_text': 'text()'
}), sink_licenses
],
],
sink1
]
run_pipeline(pipeline)
# we should have collected all the URLs to the datasets
urls = [e['url'] for e in sink1.data]
ok_(len(urls) > 20) # there should be at least 20 listed there
ok_(
all([
url.startswith('https://openfmri.org/dataset/ds00') for url in urls
]))
# got our dataset_dir entries as well
ok_(all([e['dataset_dir'].startswith('ds0') for e in sink1.data]))
# and sink2 should collect everything downloadable from under AWS Link section
# test that we got all needed tags etc propagated properly!
all_aws_entries = sink2.get_values(['dataset', 'url_text', 'url'])
ok_(len(all_aws_entries) > len(urls)) # that we have at least as many ;-)
#print('\n'.join(map(str, all_aws_entries)))
all_licenses = sink_licenses.get_values(['dataset', 'url_text', 'url'])
eq_(len(all_licenses), len(urls))
def test_check_dates(path):
skip_if_no_module("dateutil")
ref_ts = 1218182889 # Fri, 08 Aug 2008 04:08:09 -0400
refdate = "@{}".format(ref_ts)
repo = os.path.join(path, "repo")
with set_date(ref_ts + 5000):
ar = AnnexRepo(repo)
ar.add(".")
ar.commit()
# The standard renderer outputs json.
with swallow_outputs() as cmo:
# Set level to WARNING to avoid the progress bar when
# DATALAD_TESTS_UI_BACKEND=console.
with swallow_logs(new_level=logging.WARNING):
check_dates([repo],
reference_date=refdate,
return_type="list")
assert_in("report", json.loads(cmo.out).keys())
# We find the newer objects.
newer = call([path], reference_date=refdate)
eq_(len(newer), 1)
ok_(newer[0]["report"]["objects"])
# There are no older objects to find.
older = call([repo], reference_date=refdate, older=True)
assert_false(older[0]["report"]["objects"])
# We can pass the date in RFC 2822 format.
assert_dict_equal(
newer[0],
call([path], reference_date="08 Aug 2008 04:08:09 -0400")[0])
# paths=None defaults to the current directory.
with chpwd(path):
assert_dict_equal(
newer[0]["report"],
call(paths=None, reference_date=refdate)[0]["report"])
# Only commit type is present when annex='none'.
newer_noannex = call([path], reference_date=refdate, annex="none")
for entry in newer_noannex[0]["report"]["objects"].values():
ok_(entry["type"] == "commit")
def test_check_dates(path):
skip_if_no_module("dateutil")
ref_ts = 1218182889 # Fri, 08 Aug 2008 04:08:09 -0400
refdate = "@{}".format(ref_ts)
repo = os.path.join(path, "repo")
with set_date(ref_ts + 5000):
ar = AnnexRepo(repo)
ar.add(".")
ar.commit()
# The standard renderer outputs json.
with swallow_outputs() as cmo:
# Set level to WARNING to avoid the progress bar when
# DATALAD_TESTS_UI_BACKEND=console.
with swallow_logs(new_level=logging.WARNING):
check_dates([repo], reference_date=refdate, return_type="list")
assert_in("report", json.loads(cmo.out).keys())
# We find the newer objects.
newer = call([path], reference_date=refdate)
eq_(len(newer), 1)
ok_(newer[0]["report"]["objects"])
# There are no older objects to find.
older = call([repo], reference_date=refdate, older=True)
assert_false(older[0]["report"]["objects"])
# We can pass the date in RFC 2822 format.
assert_dict_equal(
newer[0],
call([path], reference_date="08 Aug 2008 04:08:09 -0400")[0])
# paths=None defaults to the current directory.
with chpwd(path):
assert_dict_equal(
newer[0]["report"],
call(paths=None, reference_date=refdate)[0]["report"])
# Only commit type is present when annex='none'.
newer_noannex = call([path], reference_date=refdate, annex="none")
for entry in newer_noannex[0]["report"]["objects"].values():
ok_(entry["type"] == "commit")
def __test_basic_openfmri_dataset_pipeline_with_annex(path):
skip_if_no_module('scrapy') # e.g. not present under Python3
dataset_index = 1
dataset_name = 'ds%06d' % dataset_index
dataset_url = 'https://openfmri.org/dataset/' + dataset_name
# needs to be a non-existing directory
dataset_path = opj(path, dataset_name)
# we need to pre-initiate dataset
list(initiate_dataset('openfmri', dataset_index, path=dataset_path)())
annex = Annexificator(
dataset_path,
create=False, # must be already initialized etc
largefiles="exclude=*.txt and exclude=README")
pipeline = [
crawl_url(dataset_url),
[ # changelog
a_href_match(".*release_history.txt"), # , limit=1
assign({'filename': 'changelog.txt'}),
annex,
],
[ # and collect all URLs under "AWS Link"
css_match('.field-name-field-aws-link a',
xpaths={
'url': '@href',
'url_text': 'text()'
}),
annex,
],
[ # and license information
css_match('.field-name-field-license a',
xpaths={
'url': '@href',
'url_text': 'text()'
}),
assign({'filename': 'license.txt'}),
annex,
],
]
run_pipeline(pipeline)
from datalad.tests.utils import skip_if_no_module
skip_if_no_module('scrapy')
from datalad.tests.utils import skip_if_scrapy_without_selector
skip_if_scrapy_without_selector()
def _test_create_store(host, base_path, ds_path, clone_path):
skip_if_no_module("ria_remote") # special remote needs to be installed
ds = Dataset(ds_path).create(force=True)
subds = ds.create('sub', force=True)
ds.save(recursive=True)
assert_repo_status(ds.path)
# don't specify special remote. By default should be git-remote + "-ria"
res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
# remotes exist, but only in super
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'here'}, {s['name'] for s in sub_siblings})
# TODO: post-update hook was enabled
# implicit test of success by ria-installing from store:
ds.publish(to="datastore", transfer_data='all')
with chpwd(clone_path):
if host:
# note, we are not using the "test-store"-label here
clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id),
path='test_install')
else:
# TODO: Whenever ria+file supports special remote config (label),
# change here:
clone('ria+file://{}#{}'.format(base_path, ds.id),
path='test_install')
installed_ds = Dataset(op.join(clone_path, 'test_install'))
assert installed_ds.is_installed()
assert_repo_status(installed_ds.repo)
eq_(installed_ds.id, ds.id)
assert_in(op.join('ds', 'file1.txt'),
installed_ds.repo.get_annexed_files())
assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')),
1,
status='ok',
action='get',
path=op.join(installed_ds.path, 'ds', 'file1.txt'))
# now, again but recursive.
res = ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
recursive=True,
existing='reconfigure')
eq_(len(res), 2)
assert_result_count(res, 2, status='ok', action="create-sibling-ria")
# remotes now exist in super and sub
siblings = ds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings})
sub_siblings = subds.siblings(result_renderer=None)
eq_({'datastore', 'datastore-ria', 'here'},
{s['name']
for s in sub_siblings})
# for testing trust_level parameter, redo for each label:
for trust in ['trust', 'semitrust', 'untrust']:
ds.create_sibling_ria("ria+ssh://test-store:",
"datastore",
existing='reconfigure',
trust_level=trust)
res = ds.repo.repo_info()
assert_in(
'[datastore-ria]',
[r['description'] for r in res['{}ed repositories'.format(trust)]])
"""Test BIDS metadata extractor """
from math import isnan
from os.path import join as opj
from simplejson import dumps
from datalad.api import Dataset
from nose.tools import assert_equal
from datalad.support.external_versions import external_versions
from datalad.tests.utils import with_tree
from datalad.tests.utils import assert_in
from datalad.tests.utils import known_failure_osx
from datalad.tests.utils import known_failure_windows
from datalad.tests.utils import skip_if_no_module
skip_if_no_module('bids')
from datalad_neuroimaging.extractors.bids import MetadataExtractor
bids_template = {
'.datalad': {
'config': '[datalad "metadata"]\n nativetype = bids',
},
'dataset_description.json': """
{
"Name": "studyforrest_phase2",
"BIDSVersion": "1.0.0-rc3",
"Description": "Some description",
"License": "PDDL",
"Authors": [
"Mike One",
def testish2():
skip_if_no_module("datalad")
return "magic"
def testish():
skip_if_no_module("nonexistingforsuremodule")
raise ValueError
def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
# Sections selection
#
# If we ask for no sections and there is no dataset
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
assert_not_in('## dataset', cmo.out)
for s in SECTION_CALLABLES:
assert_not_in('## %s' % s.lower(), cmo.out.lower())
# ask for a selected set
secs = ['git-annex', 'configuration']
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=secs)
for s in SECTION_CALLABLES:
(assert_in if s in secs else assert_not_in)(
'## %s' % s.lower(), cmo.out.lower()
)
# order should match our desired one, not alphabetical
assert cmo.out.index('## git-annex') < cmo.out.index('## configuration')
# not achievable from cmdline is to pass an empty list of sections.
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
eq_(cmo.out.rstrip(), '# WTF')
# and we could decorate it nicely for embedding e.g. into github issues
with swallow_outputs() as cmo:
wtf(sections=['dependencies'], decor='html_details')
ok_startswith(cmo.out, '<details><summary>DataLad %s WTF' % __version__)
assert_in('## dependencies', cmo.out)
# should result only in '# WTF'
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_wtf(path):
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path)
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in('path: {}'.format(ds.path), cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
# Sections selection
#
# If we ask for no sections and there is no dataset
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
assert_not_in('## dataset', cmo.out)
for s in SECTION_CALLABLES:
assert_not_in('## %s' % s.lower(), cmo.out.lower())
# ask for a selected set
secs = ['git-annex', 'configuration']
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=secs)
for s in SECTION_CALLABLES:
(assert_in if s in secs else assert_not_in)('## %s' %
s.lower(),
cmo.out.lower())
# order should match our desired one, not alphabetical
assert cmo.out.index('## git-annex') < cmo.out.index(
'## configuration')
# not achievable from cmdline is to pass an empty list of sections.
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
eq_(cmo.out.rstrip(), '# WTF')
# and we could decorate it nicely for embedding e.g. into github issues
with swallow_outputs() as cmo:
wtf(sections=['dependencies'], decor='html_details')
ok_startswith(cmo.out,
'<details><summary>DataLad %s WTF' % __version__)
assert_in('## dependencies', cmo.out)
# should result only in '# WTF'
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(exc_str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the datalad package for the
# copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
from datalad.tests.utils import skip_if_no_module
skip_if_no_module('scrapy') # e.g. not present under Python3
from datalad_neuroimaging.tests.utils import get_bids_dataset
from datalad.tests.utils import ok_clean_git
from datalad.tests.utils import with_tree
from datalad.tests.utils import eq_
from datalad.tests.utils import assert_true, assert_not_equal, assert_raises, \
assert_false, assert_equal
from datalad.tests.utils import assert_status
from datalad.tests.utils import assert_result_count
from datalad.tests.utils import assert_in
from datalad.tests.utils import with_tempfile
from datalad.support.exceptions import IncompleteResultsError
from datalad.tests.utils import skip_if_no_module
skip_if_no_module('pandas')
_dummy_template = {
'ds': {
'file_up': 'some_content',
'dir': {
'file1_down': 'one',
'file2_down': 'two'
}
}
}
_bids_template = {
'ds': {
'.datalad': {
'config':
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the datalad package for the
# copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
from datalad.tests.utils import skip_if_no_module
skip_if_no_module('scrapy') # e.g. not present under Python3