def test_wtf(path): # smoke test for now with swallow_outputs() as cmo: wtf(dataset=path) assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # Those sections get sensored out by default now assert_not_in('user.name: ', cmo.out) with chpwd(path): with swallow_outputs() as cmo: wtf() assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # now with a dataset ds = create(path) with swallow_outputs() as cmo: wtf(dataset=ds.path) assert_in('## configuration', cmo.out) assert_in('## dataset', cmo.out) assert_in('path: {}'.format(ds.path), cmo.out) # and if we run with all sensitive for sensitive in ('some', True): with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive=sensitive) # we fake those for tests anyways, but we do show cfg in this mode # and explicitly not showing them assert_in('user.name: %s' % _HIDDEN, cmo.out) with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive='all') assert_not_in(_HIDDEN, cmo.out) # all is shown assert_in('user.name: ', cmo.out) skip_if_no_module('pyperclip') # verify that it works correctly in the env/platform import pyperclip with swallow_outputs() as cmo: try: pyperclip.copy("xxx") pyperclip_works = pyperclip.paste().strip() == "xxx" wtf(dataset=ds.path, clipboard=True) except (AttributeError, pyperclip.PyperclipException) as exc: # AttributeError could come from pyperclip if no DISPLAY raise SkipTest(exc_str(exc)) assert_in("WTF information of length", cmo.out) assert_not_in('user.name', cmo.out) if not pyperclip_works: # Some times does not throw but just fails to work raise SkipTest( "Pyperclip seems to be not functioning here correctly") assert_not_in('user.name', pyperclip.paste()) assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_wtf(path): # smoke test for now with swallow_outputs() as cmo: wtf(dataset=path) assert_not_in('Dataset information', cmo.out) assert_in('Configuration', cmo.out) # Those sections get sensored out by default now assert_not_in('user.name: ', cmo.out) with chpwd(path): with swallow_outputs() as cmo: wtf() assert_not_in('Dataset information', cmo.out) assert_in('Configuration', cmo.out) # now with a dataset ds = create(path) with swallow_outputs() as cmo: wtf(dataset=ds.path) assert_in('Configuration', cmo.out) assert_in('Dataset information', cmo.out) assert_in('path: {}'.format(ds.path), cmo.out) # and if we run with all sensitive for sensitive in ('some', True): with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive=sensitive) # we fake those for tests anyways, but we do show cfg in this mode # and explicitly not showing them assert_in('user.name: %s' % _HIDDEN, cmo.out) with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive='all') assert_not_in(_HIDDEN, cmo.out) # all is shown assert_in('user.name: ', cmo.out) skip_if_no_module('pyperclip') # verify that it works correctly in the env/platform import pyperclip with swallow_outputs() as cmo: try: pyperclip.copy("xxx") pyperclip_works = pyperclip.paste().strip() == "xxx" wtf(dataset=ds.path, clipboard=True) except (AttributeError, pyperclip.PyperclipException) as exc: # AttributeError could come from pyperclip if no DISPLAY raise SkipTest(exc_str(exc)) assert_in("WTF information of length", cmo.out) assert_not_in('user.name', cmo.out) if not pyperclip_works: # Some times does not throw but just fails to work raise SkipTest( "Pyperclip seems to be not functioning here correctly") assert_not_in('user.name', pyperclip.paste()) assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info assert_in("cmd:annex=", pyperclip.paste()) # but the content is there
def test_check_dates_invalid_date(): skip_if_no_module("dateutil") with swallow_outputs() as cmo: assert_raises(IncompleteResultsError, check_dates, [], reference_date="not a valid date", return_type="list") out = cmo.out # The error makes it through the standard renderer. assert_in('"status": "error"', out)
def test_proxying_lzma_LZMAFile(): skip_if_no_module('lzma') import lzma def generate_dat(f): with LZMAFile(f, "w") as f: f.write("123".encode('utf-8')) def verify_dat(f, mode="r"): with LZMAFile(f, mode) as f: eq_(f.read().decode('utf-8'), "123") yield _test_proxying_open, generate_dat, verify_dat
def test_proxying_lzma_LZMAFile(): skip_if_no_module('datalad.support.lzma') from datalad.support.lzma import lzma def generate_dat(f): with LZMAFile(f, "w") as f: f.write("123".encode('utf-8')) def verify_dat(f, mode="r"): with LZMAFile(f, mode) as f: eq_(f.read().decode('utf-8'), "123") yield _test_proxying_open, generate_dat, verify_dat
def __test_basic_openfmri_top_pipeline(): skip_if_no_module('scrapy') # e.g. not present under Python3 sink1 = Sink() sink2 = Sink() sink_licenses = Sink() pipeline = [ crawl_url("https://openfmri.org/data-sets"), a_href_match( ".*/dataset/(?P<dataset_dir>ds0*(?P<dataset>[1-9][0-9]*))$"), # if we wanted we could instruct to crawl inside [ crawl_url(), [ # and collect all URLs under "AWS Link" css_match('.field-name-field-aws-link a', xpaths={ 'url': '@href', 'url_text': 'text()' }), sink2 ], [ # and license information css_match('.field-name-field-license a', xpaths={ 'url': '@href', 'url_text': 'text()' }), sink_licenses ], ], sink1 ] run_pipeline(pipeline) # we should have collected all the URLs to the datasets urls = [e['url'] for e in sink1.data] ok_(len(urls) > 20) # there should be at least 20 listed there ok_( all([ url.startswith('https://openfmri.org/dataset/ds00') for url in urls ])) # got our dataset_dir entries as well ok_(all([e['dataset_dir'].startswith('ds0') for e in sink1.data])) # and sink2 should collect everything downloadable from under AWS Link section # test that we got all needed tags etc propagated properly! all_aws_entries = sink2.get_values(['dataset', 'url_text', 'url']) ok_(len(all_aws_entries) > len(urls)) # that we have at least as many ;-) #print('\n'.join(map(str, all_aws_entries))) all_licenses = sink_licenses.get_values(['dataset', 'url_text', 'url']) eq_(len(all_licenses), len(urls))
def test_check_dates(path): skip_if_no_module("dateutil") ref_ts = 1218182889 # Fri, 08 Aug 2008 04:08:09 -0400 refdate = "@{}".format(ref_ts) repo = os.path.join(path, "repo") with set_date(ref_ts + 5000): ar = AnnexRepo(repo) ar.add(".") ar.commit() # The standard renderer outputs json. with swallow_outputs() as cmo: # Set level to WARNING to avoid the progress bar when # DATALAD_TESTS_UI_BACKEND=console. with swallow_logs(new_level=logging.WARNING): check_dates([repo], reference_date=refdate, return_type="list") assert_in("report", json.loads(cmo.out).keys()) # We find the newer objects. newer = call([path], reference_date=refdate) eq_(len(newer), 1) ok_(newer[0]["report"]["objects"]) # There are no older objects to find. older = call([repo], reference_date=refdate, older=True) assert_false(older[0]["report"]["objects"]) # We can pass the date in RFC 2822 format. assert_dict_equal( newer[0], call([path], reference_date="08 Aug 2008 04:08:09 -0400")[0]) # paths=None defaults to the current directory. with chpwd(path): assert_dict_equal( newer[0]["report"], call(paths=None, reference_date=refdate)[0]["report"]) # Only commit type is present when annex='none'. newer_noannex = call([path], reference_date=refdate, annex="none") for entry in newer_noannex[0]["report"]["objects"].values(): ok_(entry["type"] == "commit")
def __test_basic_openfmri_dataset_pipeline_with_annex(path): skip_if_no_module('scrapy') # e.g. not present under Python3 dataset_index = 1 dataset_name = 'ds%06d' % dataset_index dataset_url = 'https://openfmri.org/dataset/' + dataset_name # needs to be a non-existing directory dataset_path = opj(path, dataset_name) # we need to pre-initiate dataset list(initiate_dataset('openfmri', dataset_index, path=dataset_path)()) annex = Annexificator( dataset_path, create=False, # must be already initialized etc largefiles="exclude=*.txt and exclude=README") pipeline = [ crawl_url(dataset_url), [ # changelog a_href_match(".*release_history.txt"), # , limit=1 assign({'filename': 'changelog.txt'}), annex, ], [ # and collect all URLs under "AWS Link" css_match('.field-name-field-aws-link a', xpaths={ 'url': '@href', 'url_text': 'text()' }), annex, ], [ # and license information css_match('.field-name-field-license a', xpaths={ 'url': '@href', 'url_text': 'text()' }), assign({'filename': 'license.txt'}), annex, ], ] run_pipeline(pipeline)
from datalad.tests.utils import skip_if_no_module skip_if_no_module('scrapy') from datalad.tests.utils import skip_if_scrapy_without_selector skip_if_scrapy_without_selector()
def _test_create_store(host, base_path, ds_path, clone_path): skip_if_no_module("ria_remote") # special remote needs to be installed ds = Dataset(ds_path).create(force=True) subds = ds.create('sub', force=True) ds.save(recursive=True) assert_repo_status(ds.path) # don't specify special remote. By default should be git-remote + "-ria" res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) # remotes exist, but only in super siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'here'}, {s['name'] for s in sub_siblings}) # TODO: post-update hook was enabled # implicit test of success by ria-installing from store: ds.publish(to="datastore", transfer_data='all') with chpwd(clone_path): if host: # note, we are not using the "test-store"-label here clone('ria+ssh://{}{}#{}'.format(host, base_path, ds.id), path='test_install') else: # TODO: Whenever ria+file supports special remote config (label), # change here: clone('ria+file://{}#{}'.format(base_path, ds.id), path='test_install') installed_ds = Dataset(op.join(clone_path, 'test_install')) assert installed_ds.is_installed() assert_repo_status(installed_ds.repo) eq_(installed_ds.id, ds.id) assert_in(op.join('ds', 'file1.txt'), installed_ds.repo.get_annexed_files()) assert_result_count(installed_ds.get(op.join('ds', 'file1.txt')), 1, status='ok', action='get', path=op.join(installed_ds.path, 'ds', 'file1.txt')) # now, again but recursive. res = ds.create_sibling_ria("ria+ssh://test-store:", "datastore", recursive=True, existing='reconfigure') eq_(len(res), 2) assert_result_count(res, 2, status='ok', action="create-sibling-ria") # remotes now exist in super and sub siblings = ds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in siblings}) sub_siblings = subds.siblings(result_renderer=None) eq_({'datastore', 'datastore-ria', 'here'}, {s['name'] for s in sub_siblings}) # for testing trust_level parameter, redo for each label: for trust in ['trust', 'semitrust', 'untrust']: ds.create_sibling_ria("ria+ssh://test-store:", "datastore", existing='reconfigure', trust_level=trust) res = ds.repo.repo_info() assert_in( '[datastore-ria]', [r['description'] for r in res['{}ed repositories'.format(trust)]])
"""Test BIDS metadata extractor """ from math import isnan from os.path import join as opj from simplejson import dumps from datalad.api import Dataset from nose.tools import assert_equal from datalad.support.external_versions import external_versions from datalad.tests.utils import with_tree from datalad.tests.utils import assert_in from datalad.tests.utils import known_failure_osx from datalad.tests.utils import known_failure_windows from datalad.tests.utils import skip_if_no_module skip_if_no_module('bids') from datalad_neuroimaging.extractors.bids import MetadataExtractor bids_template = { '.datalad': { 'config': '[datalad "metadata"]\n nativetype = bids', }, 'dataset_description.json': """ { "Name": "studyforrest_phase2", "BIDSVersion": "1.0.0-rc3", "Description": "Some description", "License": "PDDL", "Authors": [ "Mike One",
def testish2(): skip_if_no_module("datalad") return "magic"
def testish(): skip_if_no_module("nonexistingforsuremodule") raise ValueError
def test_wtf(path): # smoke test for now with swallow_outputs() as cmo: wtf(dataset=path) assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # Those sections get sensored out by default now assert_not_in('user.name: ', cmo.out) with chpwd(path): with swallow_outputs() as cmo: wtf() assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # now with a dataset ds = create(path) with swallow_outputs() as cmo: wtf(dataset=ds.path) assert_in('## configuration', cmo.out) assert_in('## dataset', cmo.out) assert_in('path: {}'.format(ds.path), cmo.out) # and if we run with all sensitive for sensitive in ('some', True): with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive=sensitive) # we fake those for tests anyways, but we do show cfg in this mode # and explicitly not showing them assert_in('user.name: %s' % _HIDDEN, cmo.out) with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive='all') assert_not_in(_HIDDEN, cmo.out) # all is shown assert_in('user.name: ', cmo.out) # Sections selection # # If we ask for no sections and there is no dataset with chpwd(path): with swallow_outputs() as cmo: wtf(sections=[]) assert_not_in('## dataset', cmo.out) for s in SECTION_CALLABLES: assert_not_in('## %s' % s.lower(), cmo.out.lower()) # ask for a selected set secs = ['git-annex', 'configuration'] with chpwd(path): with swallow_outputs() as cmo: wtf(sections=secs) for s in SECTION_CALLABLES: (assert_in if s in secs else assert_not_in)( '## %s' % s.lower(), cmo.out.lower() ) # order should match our desired one, not alphabetical assert cmo.out.index('## git-annex') < cmo.out.index('## configuration') # not achievable from cmdline is to pass an empty list of sections. with chpwd(path): with swallow_outputs() as cmo: wtf(sections=[]) eq_(cmo.out.rstrip(), '# WTF') # and we could decorate it nicely for embedding e.g. into github issues with swallow_outputs() as cmo: wtf(sections=['dependencies'], decor='html_details') ok_startswith(cmo.out, '<details><summary>DataLad %s WTF' % __version__) assert_in('## dependencies', cmo.out) # should result only in '# WTF' skip_if_no_module('pyperclip') # verify that it works correctly in the env/platform import pyperclip with swallow_outputs() as cmo: try: pyperclip.copy("xxx") pyperclip_works = pyperclip.paste().strip() == "xxx" wtf(dataset=ds.path, clipboard=True) except (AttributeError, pyperclip.PyperclipException) as exc: # AttributeError could come from pyperclip if no DISPLAY raise SkipTest(exc_str(exc)) assert_in("WTF information of length", cmo.out) assert_not_in('user.name', cmo.out) if not pyperclip_works: # Some times does not throw but just fails to work raise SkipTest( "Pyperclip seems to be not functioning here correctly") assert_not_in('user.name', pyperclip.paste()) assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def test_wtf(path): # smoke test for now with swallow_outputs() as cmo: wtf(dataset=path) assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # Those sections get sensored out by default now assert_not_in('user.name: ', cmo.out) with chpwd(path): with swallow_outputs() as cmo: wtf() assert_not_in('## dataset', cmo.out) assert_in('## configuration', cmo.out) # now with a dataset ds = create(path) with swallow_outputs() as cmo: wtf(dataset=ds.path) assert_in('## configuration', cmo.out) assert_in('## dataset', cmo.out) assert_in('path: {}'.format(ds.path), cmo.out) # and if we run with all sensitive for sensitive in ('some', True): with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive=sensitive) # we fake those for tests anyways, but we do show cfg in this mode # and explicitly not showing them assert_in('user.name: %s' % _HIDDEN, cmo.out) with swallow_outputs() as cmo: wtf(dataset=ds.path, sensitive='all') assert_not_in(_HIDDEN, cmo.out) # all is shown assert_in('user.name: ', cmo.out) # Sections selection # # If we ask for no sections and there is no dataset with chpwd(path): with swallow_outputs() as cmo: wtf(sections=[]) assert_not_in('## dataset', cmo.out) for s in SECTION_CALLABLES: assert_not_in('## %s' % s.lower(), cmo.out.lower()) # ask for a selected set secs = ['git-annex', 'configuration'] with chpwd(path): with swallow_outputs() as cmo: wtf(sections=secs) for s in SECTION_CALLABLES: (assert_in if s in secs else assert_not_in)('## %s' % s.lower(), cmo.out.lower()) # order should match our desired one, not alphabetical assert cmo.out.index('## git-annex') < cmo.out.index( '## configuration') # not achievable from cmdline is to pass an empty list of sections. with chpwd(path): with swallow_outputs() as cmo: wtf(sections=[]) eq_(cmo.out.rstrip(), '# WTF') # and we could decorate it nicely for embedding e.g. into github issues with swallow_outputs() as cmo: wtf(sections=['dependencies'], decor='html_details') ok_startswith(cmo.out, '<details><summary>DataLad %s WTF' % __version__) assert_in('## dependencies', cmo.out) # should result only in '# WTF' skip_if_no_module('pyperclip') # verify that it works correctly in the env/platform import pyperclip with swallow_outputs() as cmo: try: pyperclip.copy("xxx") pyperclip_works = pyperclip.paste().strip() == "xxx" wtf(dataset=ds.path, clipboard=True) except (AttributeError, pyperclip.PyperclipException) as exc: # AttributeError could come from pyperclip if no DISPLAY raise SkipTest(exc_str(exc)) assert_in("WTF information of length", cmo.out) assert_not_in('user.name', cmo.out) if not pyperclip_works: # Some times does not throw but just fails to work raise SkipTest( "Pyperclip seems to be not functioning here correctly") assert_not_in('user.name', pyperclip.paste()) assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- # ex: set sts=4 ts=4 sw=4 noet: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the datalad package for the # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## from datalad.tests.utils import skip_if_no_module skip_if_no_module('scrapy') # e.g. not present under Python3
from datalad_neuroimaging.tests.utils import get_bids_dataset from datalad.tests.utils import ok_clean_git from datalad.tests.utils import with_tree from datalad.tests.utils import eq_ from datalad.tests.utils import assert_true, assert_not_equal, assert_raises, \ assert_false, assert_equal from datalad.tests.utils import assert_status from datalad.tests.utils import assert_result_count from datalad.tests.utils import assert_in from datalad.tests.utils import with_tempfile from datalad.support.exceptions import IncompleteResultsError from datalad.tests.utils import skip_if_no_module skip_if_no_module('pandas') _dummy_template = { 'ds': { 'file_up': 'some_content', 'dir': { 'file1_down': 'one', 'file2_down': 'two' } } } _bids_template = { 'ds': { '.datalad': { 'config':