def eeg_data_path(base_path, subject, accept):
datapath = op.join(base_path, "EEG", "subject {:02d}".format(subject),
"with occular artifact")
if not op.isfile(op.join(datapath, "cnt.mat")):
if not op.isdir(op.join(base_path, "EEG")):
os.makedirs(op.join(base_path, "EEG"))
intervals = [[1, 5], [6, 10], [11, 15], [16, 20], [21, 25], [26, 29]]
for low, high in intervals:
if subject >= low and subject <= high:
if not op.isfile(op.join(base_path, "EEG.zip")):
if not accept:
raise AttributeError(
"You must accept licence term to download this dataset,"
"set accept=True when instanciating the dataset.")
_fetch_file(
"{}/EEG/EEG_{:02d}-{:02d}.zip".format(
SHIN_URL, low, high),
op.join(base_path, "EEG.zip"),
print_destination=False,
)
with z.ZipFile(op.join(base_path, "EEG.zip"), "r") as f:
f.extractall(op.join(base_path, "EEG"))
os.remove(op.join(base_path, "EEG.zip"))
break
assert op.isfile(op.join(datapath,
"cnt.mat")), op.join(datapath, "cnt.mat")
return [op.join(datapath, fn) for fn in ["cnt.mat", "mrk.mat"]]
def test_fetch_file():
"""Test file downloading
"""
tempdir = _TempDir()
urls = [
'http://martinos.org/mne/',
'ftp://surfer.nmr.mgh.harvard.edu/pub/data/bert.recon.md5sum.txt'
]
with ArgvSetter(disable_stderr=False): # to capture stdout
for url in urls:
archive_name = op.join(tempdir, "download_test")
_fetch_file(url, archive_name, verbose=False)
assert_raises(Exception,
_fetch_file,
'NOT_AN_ADDRESS',
op.join(tempdir, 'test'),
verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, resume=True, verbose=False)
assert_raises(ValueError,
_fetch_file,
url,
archive_name,
hash_='a',
verbose=False)
assert_raises(RuntimeError,
_fetch_file,
url,
archive_name,
hash_='a' * 32,
verbose=False)
def _test_fetch(url):
"""Test URL retrieval."""
tempdir = _TempDir()
with ArgvSetter(disable_stderr=False): # to capture stdout
archive_name = op.join(tempdir, "download_test")
_fetch_file(url,
archive_name,
timeout=30.,
verbose=False,
resume=False)
pytest.raises(Exception,
_fetch_file,
'NOT_AN_ADDRESS',
op.join(tempdir, 'test'),
verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, resume=True, timeout=30., verbose=False)
pytest.raises(ValueError,
_fetch_file,
url,
archive_name,
hash_='a',
verbose=False)
pytest.raises(RuntimeError,
_fetch_file,
url,
archive_name,
hash_='a' * 32,
verbose=False)
def fetch_faces_data(data_path, repo, subject_ids):
"""Dataset fetcher for OpenfMRI dataset ds000117.
Parameters
----------
data_path : str
Path to the folder where data is stored
repo : str
The folder name (typically 'ds000117')
subject_ids : list of int
The subjects to fetch
"""
for subject_id in subject_ids:
src_url = ('http://openfmri.s3.amazonaws.com/tarballs/'
'ds117_R0.1.1_sub%03d_raw.tgz' % subject_id)
tar_fname = op.join(data_path, repo + '.tgz')
target_dir = op.join(data_path, repo)
if not op.exists(target_dir):
if not op.exists(tar_fname):
_fetch_file(url=src_url,
file_name=tar_fname,
print_destination=True,
resume=True,
timeout=10.)
tf = tarfile.open(tar_fname)
print('Extracting files. This may take a while ...')
tf.extractall(path=data_path)
shutil.move(op.join(data_path, 'ds117'), target_dir)
os.remove(tar_fname)
def download_file(url, name, root_destination='~/data/', zipfile=False,
replace=False):
"""Download a file from dropbox, google drive, or a URL.
This will download a file and store it in a '~/data/` folder,
creating directories if need be. It will also work for zip
files, in which case it will unzip all of the files to the
desired location.
Parameters
----------
url : string
The url of the file to download. This may be a dropbox
or google drive "share link", or a regular URL. If it
is a share link, then it should point to a single file and
not a folder. To download folders, zip them first.
name : string
The name / path of the file for the downloaded file, or
the folder to zip the data into if the file is a zipfile.
root_destination : string
The root folder where data will be downloaded.
zipfile : bool
Whether the URL points to a zip file. If yes, it will be
unzipped to root_destination + name.
replace : bool
If True and the URL points to a single file, overwrite the
old file if possible.
"""
# Make sure we have directories to dump files
home = op.expanduser('~')
tmpfile = home + '/tmp/tmp'
if not op.isdir(home + '/data/'):
print('Creating data folder...')
os.makedirs(home + '/data/')
if not op.isdir(home + '/tmp/'):
print('Creating tmp folder...')
os.makedirs(home + '/tmp/')
download_path = _convert_url_to_downloadable(url)
# Now save to the new destination
out_path = root_destination.replace('~', home) + name
if not op.isdir(op.dirname(out_path)):
print('Creating path {} for output data'.format(out_path))
os.makedirs(op.dirname(out_path))
if zipfile is True:
_fetch_file(download_path, tmpfile)
myzip = ZipFile(tmpfile)
myzip.extractall(out_path)
os.remove(tmpfile)
else:
if len(name) == 0:
raise ValueError('Cannot overwrite the root data directory')
if replace is False and op.exists(out_path):
raise ValueError('Path {} exists, use `replace=True` to '
'overwrite'.format(out_path))
_fetch_file(download_path, out_path)
print('Successfully moved file to {}'.format(out_path))
def data_path(url,
sign,
path=None,
force_update=False,
update_path=True,
verbose=None):
"""Get path to local copy of given dataset URL.
This is a low-level function useful for getting a local copy of a
remote dataset
Parameters
----------
url : str
Path to remote location of data
sign : str
Signifier of dataset
path : None | str
Location of where to look for the BNCI data storing location.
If None, the environment variable or config parameter
``MNE_DATASETS_(signifier)_PATH`` is used. If it doesn't exist, the
"~/mne_data" directory is used. If the dataset
is not found under the given path, the data
will be automatically downloaded to the specified folder.
force_update : bool
Force update of the dataset even if a local copy exists.
update_path : bool | None
If True, set the MNE_DATASETS_(signifier)_PATH in mne-python
config to the given path. If None, the user is prompted.
verbose : bool, str, int, or None
If not None, override default verbose level (see :func:`mne.verbose`).
Returns
-------
path : list of str
Local path to the given data file. This path is contained inside a list
of length one, for compatibility.
""" # noqa: E501
sign = sign.upper()
key = 'MNE_DATASETS_{:s}_PATH'.format(sign)
key_dest = 'MNE-{:s}-data'.format(sign.lower())
if get_config(key) is None:
set_config(key, osp.join(osp.expanduser("~"), "mne_data"))
path = _get_path(path, key, sign)
destination = _url_to_local_path(url, op.join(path, key_dest))
# Fetch the file
if not op.isfile(destination) or force_update:
if op.isfile(destination):
os.remove(destination)
if not op.isdir(op.dirname(destination)):
os.makedirs(op.dirname(destination))
_fetch_file(url, destination, print_destination=False)
# Offer to update the path
_do_path_update(path, update_path, key, sign)
return destination
def run_hnn_core(backend=None, n_jobs=1):
"""Test to check if hnn-core does not break."""
# small snippet of data on data branch for now. To be deleted
# later. Data branch should have only commit so it does not
# pollute the history.
data_url = ('https://raw.githubusercontent.com/jonescompneurolab/'
'hnn-core/test_data/dpl.txt')
if not op.exists('dpl.txt'):
_fetch_file(data_url, 'dpl.txt')
dpl_master = loadtxt('dpl.txt')
hnn_core_root = op.dirname(hnn_core.__file__)
# default params
params_fname = op.join(hnn_core_root, 'param', 'default.json')
params = read_params(params_fname)
# run the simulation
net = Network(params)
if backend == 'mpi':
with MPIBackend(n_procs=2, mpi_cmd='mpiexec'):
dpl = simulate_dipole(net)[0]
elif backend == 'joblib':
with JoblibBackend(n_jobs=n_jobs):
dpl = simulate_dipole(net)[0]
else:
dpl = simulate_dipole(net)[0]
# write the dipole to a file and compare
fname = './dpl2.txt'
dpl.write(fname)
dpl_pr = loadtxt(fname)
assert_array_equal(dpl_pr[:, 2], dpl_master[:, 2]) # L2
assert_array_equal(dpl_pr[:, 3], dpl_master[:, 3]) # L5
# Test spike type counts
spiketype_counts = {}
for spikegid in net.spikes.gids[0]:
if net.gid_to_type(spikegid) not in spiketype_counts:
spiketype_counts[net.gid_to_type(spikegid)] = 0
else:
spiketype_counts[net.gid_to_type(spikegid)] += 1
assert 'common' not in spiketype_counts
assert 'exgauss' not in spiketype_counts
assert 'extpois' not in spiketype_counts
assert spiketype_counts == {
'evprox1': 269,
'L2_basket': 54,
'L2_pyramidal': 113,
'L5_pyramidal': 395,
'L5_basket': 85,
'evdist1': 234,
'evprox2': 269
}
def test_hnn():
"""Test to check that HNN produces consistent results"""
# small snippet of data on data branch for now. To be deleted
# later. Data branch should have only commit so it does not
# pollute the history.
from subprocess import Popen, PIPE
import shlex
import os
import sys
ntrials = 3
paramf = op.join('param', 'default.param')
nrniv_str = 'nrniv -python -nobanner'
cmd = nrniv_str + ' ' + sys.executable + ' run.py ' + paramf + ' ntrial ' + str(ntrials)
# Split the command into shell arguments for passing to Popen
cmdargs = shlex.split(cmd, posix="win" not in sys.platform)
# Start the simulation
proc = Popen(cmdargs, stdin=PIPE, stdout=PIPE, stderr=PIPE,
cwd=os.getcwd(), universal_newlines=True)
out, err = proc.communicate()
# print all messages (including error messages)
print('STDOUT', out)
print('STDERR', err)
for trial in range(ntrials):
print("Checking data for trial %d" % trial)
if 'SYSTEM_USER_DIR' in os.environ:
basedir = os.environ['SYSTEM_USER_DIR']
else:
basedir = os.path.expanduser('~')
dirname = op.join(basedir, 'hnn_out', 'data', 'default')
data_dir = ('https://raw.githubusercontent.com/jonescompneurolab/'
'hnn/test_data/')
for data_type in [ 'dpl', 'rawdpl', 'i']:
sys.stdout.write("%s..." % data_type)
fname = "%s_%d.txt" % (data_type, trial)
data_url = op.join(data_dir, fname)
if not op.exists(fname):
_fetch_file(data_url, fname)
print("comparing %s" % fname)
pr = loadtxt(op.join(dirname, fname))
master = loadtxt(fname)
assert_allclose(pr[:, 1], master[:, 1], rtol=1e-8, atol=0)
if data_type in ['dpl', 'rawdpl', 'i']:
assert_allclose(pr[:, 2], master[:, 2], rtol=1e-8, atol=0)
if data_type in ['dpl', 'rawdpl']:
assert_allclose(pr[:, 3], master[:, 3], rtol=1e-8, atol=0)
print("done")
def fetch_faces_data(data_path=None, repo='ds000117', subject_ids=[1]):
"""Dataset fetcher for OpenfMRI dataset ds000117.
Parameters
----------
data_path : str | None
Path to the folder where data is stored. Defaults to
'~/mne_data/mne_bids_examples'
repo : str
The folder name. Defaults to 'ds000117'.
subject_ids : list of int
The subjects to fetch. Defaults to [1], downloading subject 1.
Returns
-------
data_path : str
Path to the parent directory where the `repo` folder containing the
data is stored.
"""
if data_path is None:
home = os.path.expanduser('~')
data_path = os.path.join(home, 'mne_data', 'mne_bids_examples')
if not os.path.exists(data_path):
os.makedirs(data_path)
target_dir = op.join(data_path, repo)
os.makedirs(target_dir, exist_ok=True)
for subject_id in subject_ids: # pragma: no cover
# If we have data for that subject already, skip
sub_str = 'sub{:03}'.format(subject_id)
if op.exists(op.join(target_dir, sub_str)):
continue
# Else, download it
src_url = ('http://openfmri.s3.amazonaws.com/tarballs/'
'ds117_R0.1.1_{}_raw.tgz'.format(sub_str))
tar_fname = op.join(data_path, repo, sub_str + '.tgz')
if not op.exists(tar_fname):
_fetch_file(url=src_url, file_name=tar_fname,
print_destination=True)
# Unpack the downloaded archive to the correct location
tf = tarfile.open(tar_fname)
print('Extracting files. This may take a while ...')
tf.extractall(path=data_path) # will extract the 'ds117' directory
shutil.move(op.join(data_path, 'ds117', sub_str), target_dir)
# Clean up to get ready to download next subject
shutil.rmtree(op.join(data_path, 'ds117'))
os.remove(tar_fname)
return data_path
def test_base_params():
"""Test default params object matches base params"""
param_url = ('https://raw.githubusercontent.com/jonescompneurolab/'
'hnn-core/test_data/base.json')
params_base_fname = op.join(hnn_core_root, 'param', 'base.json')
if not op.exists(params_base_fname):
_fetch_file(param_url, params_base_fname)
params_base = read_params(params_base_fname)
params = Params()
assert params == params_base
def load(name):
if name in names:
fname = os.path.join(dirname, f"{name}.pickled")
if not os.path.isfile(fname):
_fetch_file(url % name, fname)
else:
print(f"{name}.pickled already downloaded.")
else:
raise ValueError(f"{name}: not found")
with open(fname, 'rb') as f:
v = pickle.load(f)
return v
def _data_path(path=None, force_update=False, update_path=True,
download=True, name=None, verbose=None):
if path is None:
path = op.join(os.environ["COMA_DIR"],"examples")
if not isinstance(path, basestring):
raise ValueError('path must be a string or None')
if name == 'example':
archive_name = "ComaSample.tar.gz"
url = "https://www.dropbox.com/s/thpu5cph0hv94bz/" + archive_name + "?dl=1"
folder_name = "ComaSample"
folder_path = op.join(path, folder_name)
rm_archive = False
else:
raise ValueError('Sorry, the dataset "%s" is not known.' % name)
if not op.exists(folder_path) and not download:
return ''
if not op.exists(folder_path) or force_update:
logger.info('Sample data archive %s not found at:\n%s\n'
'It will be downloaded and extracted at this location.'
% (archive_name, folder_path))
archive_name = op.join(path, archive_name)
rm_archive = True
if op.exists(archive_name):
msg = ('Archive already exists at %r. Overwrite it '
'(y/[n])? ' % archive_name)
answer = raw_input(msg)
if answer.lower() == 'y':
os.remove(archive_name)
else:
raise IOError('Archive file already exists at target '
'location %r.' % archive_name)
_fetch_file(url, archive_name, print_destination=False)
if op.exists(folder_path):
shutil.rmtree(folder_path)
logger.info('Decompressing the archive: ' + archive_name)
logger.info('... please be patient, this can take some time')
for ext in ['gz', 'bz2']: # informed guess (and the only 2 options).
try:
tarfile.open(archive_name, 'r:%s' % ext).extractall(path=path)
except tarfile.ReadError, err:
logger.info('%s is %s trying "bz2"' % (archive_name, err))
if rm_archive:
os.remove(archive_name)
def test_legacy_params():
"""Test reading legacy .param file."""
param_url = ('https://raw.githubusercontent.com/hnnsolver/'
'hnn-core/test_data/default.param')
params_legacy_fname = op.join(hnn_core_root, 'param', 'default.param')
if not op.exists(params_legacy_fname):
_fetch_file(param_url, params_legacy_fname)
params_new_fname = op.join(hnn_core_root, 'param', 'default.json')
params_legacy = read_params(params_legacy_fname)
params_new = read_params(params_new_fname)
assert params_new == params_legacy
def fnirs_data_path(path, subject):
datapath = op.join(path, 'NIRS', 'subject {:02d}'.format(subject))
if not op.isfile(op.join(datapath, 'mrk.mat')):
# fNIRS
if not op.isfile(op.join(path, 'fNIRS.zip')):
_fetch_file('http://doc.ml.tu-berlin.de/hBCI/NIRS/NIRS_01-29.zip',
op.join(path, 'fNIRS.zip'),
print_destination=False)
if not op.isdir(op.join(path, 'NIRS')):
os.makedirs(op.join(path, 'NIRS'))
with z.ZipFile(op.join(path, 'fNIRS.zip'), 'r') as f:
f.extractall(op.join(path, 'NIRS'))
os.remove(op.join(path, 'fNIRS.zip'))
return [op.join(datapath, fn) for fn in ['cnt.mat', 'mrk.mat']]
def download_test_data():
'''Download additional test data from dropbox.'''
import zipfile
try:
from mne.utils import _fetch_file
use_pooch = False
except ImportError:
import pooch
use_pooch = True
# check if test data exist
data_dir = _get_test_data_dir()
check_files = [
'alpha_range_clusters.hdf5', 'DiamSar-eeg-oct-6-fwd.fif',
op.join('fsaverage', 'bem',
'fsaverage-ico-5-src.fif'), 'chan_alpha_range.hdf5',
'test_clustering.npy', 'DiamSar_023_rest_raw.fif', 'GabCon-48_epo.fif'
]
if all([op.isfile(op.join(data_dir, f)) for f in check_files]):
return
# set up paths
fname = 'temp_file.zip'
destination = op.join(data_dir, fname)
download_link = ('https://www.dropbox.com/sh/l4scs37524lb3pa/'
'AABCak4jORjgridWwHlwjhMHa?dl=1')
# download the file
if use_pooch:
hash = ('98bab9750844ab969c5a74deea9d041701d73f43dfe05465c'
'577a83d974064e8')
pooch.retrieve(url=download_link,
known_hash=hash,
path=data_dir,
fname=fname)
else:
_fetch_file(download_link,
destination,
print_destination=True,
resume=True,
timeout=30.)
# unzip and extract
# TODO - optionally extract only the missing files
zip_ref = zipfile.ZipFile(destination, 'r')
zip_ref.extractall(data_dir)
zip_ref.close()
# remove the zipfile
os.remove(destination)
def fnirs_data_path(path, subject):
datapath = op.join(path, "NIRS", "subject {:02d}".format(subject))
if not op.isfile(op.join(datapath, "mrk.mat")):
# fNIRS
if not op.isfile(op.join(path, "fNIRS.zip")):
_fetch_file(
"http://doc.ml.tu-berlin.de/hBCI/NIRS/NIRS_01-29.zip",
op.join(path, "fNIRS.zip"),
print_destination=False,
)
if not op.isdir(op.join(path, "NIRS")):
os.makedirs(op.join(path, "NIRS"))
with z.ZipFile(op.join(path, "fNIRS.zip"), "r") as f:
f.extractall(op.join(path, "NIRS"))
os.remove(op.join(path, "fNIRS.zip"))
return [op.join(datapath, fn) for fn in ["cnt.mat", "mrk.mat"]]
def test_fetch_file():
"""Test file downloading
"""
tempdir = _TempDir()
urls = ["http://martinos.org/mne/", "ftp://surfer.nmr.mgh.harvard.edu/pub/data/bert.recon.md5sum.txt"]
with ArgvSetter(disable_stderr=False): # to capture stdout
for url in urls:
archive_name = op.join(tempdir, "download_test")
_fetch_file(url, archive_name, verbose=False)
assert_raises(Exception, _fetch_file, "NOT_AN_ADDRESS", op.join(tempdir, "test"), verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + ".part", "w"):
os.utime(resume_name + ".part", None)
_fetch_file(url, resume_name, resume=True, verbose=False)
assert_raises(ValueError, _fetch_file, url, archive_name, hash_="a", verbose=False)
assert_raises(RuntimeError, _fetch_file, url, archive_name, hash_="a" * 32, verbose=False)
def fetch_file(url, file_name, resume=True, timeout=10.):
"""Load requested file, downloading it if needed or requested.
Parameters
----------
url: string
The url of file to be downloaded.
file_name: string
Name, along with the path, of where downloaded file will be saved.
resume: bool, optional
If true, try to resume partially downloaded files.
timeout : float
The URL open timeout.
"""
from mne.utils import _fetch_file
_fetch_file(url=url, file_name=file_name, print_destination=True,
resume=resume, hash_=None, timeout=timeout)
def fetch_file(url, file_name, resume=True, timeout=10.):
"""Load requested file, downloading it if needed or requested
Parameters
----------
url: string
The url of file to be downloaded.
file_name: string
Name, along with the path, of where downloaded file will be saved.
resume: bool, optional
If true, try to resume partially downloaded files.
timeout : float
The URL open timeout.
"""
from mne.utils import _fetch_file
_fetch_file(url=url, file_name=file_name, print_destination=True,
resume=resume, hash_=None, timeout=timeout)
def test_hnn_core():
"""Test to check if MNE neuron does not break."""
# small snippet of data on data branch for now. To be deleted
# later. Data branch should have only commit so it does not
# pollute the history.
data_url = ('https://raw.githubusercontent.com/jonescompneurolab/'
'hnn-core/test_data/dpl.txt')
if not op.exists('dpl.txt'):
_fetch_file(data_url, 'dpl.txt')
dpl_master = loadtxt('dpl.txt')
hnn_core_root = op.join(op.dirname(hnn_core.__file__), '..')
params_fname = op.join(hnn_core_root, 'param', 'default.json')
params = read_params(params_fname)
net = Network(params, n_jobs=1)
dpl = simulate_dipole(net)[0]
fname = './dpl2.txt'
dpl.write(fname)
dpl_pr = loadtxt(fname)
assert_array_equal(dpl_pr[:, 2], dpl_master[:, 2]) # L2
assert_array_equal(dpl_pr[:, 3], dpl_master[:, 3]) # L5
# Test spike type counts
spiketype_counts = {}
for spikegid in net.spikegids[0]:
if net.gid_to_type(spikegid) not in spiketype_counts:
spiketype_counts[net.gid_to_type(spikegid)] = 0
else:
spiketype_counts[net.gid_to_type(spikegid)] += 1
assert 'extinput' not in spiketype_counts
assert 'exgauss' not in spiketype_counts
assert 'extpois' not in spiketype_counts
assert spiketype_counts == {
'evprox1': 269,
'L2_basket': 54,
'L2_pyramidal': 113,
'L5_pyramidal': 395,
'L5_basket': 85,
'evdist1': 234,
'evprox2': 269
}
def get_subjects(sub_inds, sub_names, ind):
dataname = 'data{}'.format(ind)
if not os.path.isfile(os.path.join(base_path, dataname + '.zip')):
_fetch_file(FILES[ind],
os.path.join(base_path, dataname + '.zip'),
print_destination=False)
with z.ZipFile(os.path.join(base_path, dataname + '.zip'), 'r') as f:
os.makedirs(os.path.join(base_path, dataname), exist_ok=True)
f.extractall(os.path.join(base_path, dataname))
for fname in os.listdir(os.path.join(base_path, dataname)):
for ind, prefix in zip(sub_inds, sub_names):
if fname.startswith(prefix):
os.rename(
os.path.join(base_path, dataname, fname),
os.path.join(base_path,
'subject_{}.mat'.format(ind)))
os.remove(os.path.join(base_path, dataname + '.zip'))
shutil.rmtree(os.path.join(base_path, dataname))
def data_path(url, sign, path=None, force_update=False, update_path=True, verbose=None):
"""Get path to local copy of given dataset URL. **Deprecated**
This is a low-level function useful for getting a local copy of a
remote dataset. It is deprecated in favor of data_dl.
Parameters
----------
url : str
Path to remote location of data
sign : str
Signifier of dataset
path : None | str
Location of where to look for the data storing location.
If None, the environment variable or config parameter
``MNE_DATASETS_(signifier)_PATH`` is used. If it doesn't exist, the
"~/mne_data" directory is used. If the dataset
is not found under the given path, the data
will be automatically downloaded to the specified folder.
force_update : bool
Force update of the dataset even if a local copy exists.
update_path : bool | None, **Deprecated**
Unused, kept for compatibility purpose.
verbose : bool, str, int, or None
If not None, override default verbose level (see :func:`mne.verbose`).
Returns
-------
path : list of str
Local path to the given data file. This path is contained inside a list
of length one, for compatibility.
""" # noqa: E501
path = get_dataset_path(sign, path)
key_dest = "MNE-{:s}-data".format(sign.lower())
destination = _url_to_local_path(url, osp.join(path, key_dest))
# Fetch the file
if not osp.isfile(destination) or force_update:
if osp.isfile(destination):
os.remove(destination)
if not osp.isdir(osp.dirname(destination)):
os.makedirs(osp.dirname(destination))
_fetch_file(url, destination, print_destination=False)
return destination
def fetch_faces_data(data_path=None, repo='ds000117', subject_ids=[1]):
"""Dataset fetcher for OpenfMRI dataset ds000117.
Parameters
----------
data_path : str | None
Path to the folder where data is stored. Defaults to
'~/mne_data/mne_bids_examples'
repo : str
The folder name. Defaults to 'ds000117'.
subject_ids : list of int
The subjects to fetch. Defaults to [1], downloading subject 1.
Returns
-------
data_path : str
Path to the folder where data is stored.
"""
if not data_path:
home = os.path.expanduser('~')
data_path = os.path.join(home, 'mne_data', 'mne_bids_examples')
if not os.path.exists(data_path):
os.makedirs(data_path)
for subject_id in subject_ids:
src_url = ('http://openfmri.s3.amazonaws.com/tarballs/'
'ds117_R0.1.1_sub%03d_raw.tgz' % subject_id)
tar_fname = op.join(data_path, repo + '.tgz')
target_dir = op.join(data_path, repo)
if not op.exists(target_dir):
if not op.exists(tar_fname):
_fetch_file(url=src_url,
file_name=tar_fname,
print_destination=True,
resume=True,
timeout=10.)
tf = tarfile.open(tar_fname)
print('Extracting files. This may take a while ...')
tf.extractall(path=data_path)
shutil.move(op.join(data_path, 'ds117'), target_dir)
os.remove(tar_fname)
return data_path
def _test_fetch(url):
"""Helper to test URL retrieval."""
tempdir = _TempDir()
with ArgvSetter(disable_stderr=False): # to capture stdout
archive_name = op.join(tempdir, "download_test")
_fetch_file(url, archive_name, timeout=30., verbose=False,
resume=False)
assert_raises(Exception, _fetch_file, 'NOT_AN_ADDRESS',
op.join(tempdir, 'test'), verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, resume=True, timeout=30.,
verbose=False)
assert_raises(ValueError, _fetch_file, url, archive_name,
hash_='a', verbose=False)
assert_raises(RuntimeError, _fetch_file, url, archive_name,
hash_='a' * 32, verbose=False)
def fnirs_data_path(path, subject, accept):
datapath = op.join(path, "NIRS", "subject {:02d}".format(subject))
if not op.isfile(op.join(datapath, "mrk.mat")):
# fNIRS
if not op.isfile(op.join(path, "fNIRS.zip")):
if not accept:
raise AttributeError(
"You must accept licence term to download this dataset,"
"set accept=True when instanciating the dataset.")
_fetch_file(
"http://doc.ml.tu-berlin.de/hBCI/NIRS/NIRS_01-29.zip",
op.join(path, "fNIRS.zip"),
print_destination=False,
)
if not op.isdir(op.join(path, "NIRS")):
os.makedirs(op.join(path, "NIRS"))
with z.ZipFile(op.join(path, "fNIRS.zip"), "r") as f:
f.extractall(op.join(path, "NIRS"))
os.remove(op.join(path, "fNIRS.zip"))
return [op.join(datapath, fn) for fn in ["cnt.mat", "mrk.mat"]]
def eeg_data_path(base_path, subject):
datapath = op.join(base_path, 'EEG', 'subject {:02d}'.format(
subject), 'with occular artifact')
if not op.isfile(op.join(datapath, 'cnt.mat')):
if not op.isdir(op.join(base_path, 'EEG')):
os.makedirs(op.join(base_path, 'EEG'))
intervals = [[1, 5], [6, 10], [11, 15], [16, 20], [21, 25], [26, 29]]
for low, high in intervals:
if subject >= low and subject <= high:
if not op.isfile(op.join(base_path, 'EEG.zip')):
_fetch_file('http://doc.ml.tu-berlin.de/hBCI/EEG/EEG_{:02d}-{:02d}.zip'.format(low,
high),
op.join(base_path, 'EEG.zip'), print_destination=False)
with z.ZipFile(op.join(base_path, 'EEG.zip'), 'r') as f:
f.extractall(op.join(base_path, 'EEG'))
os.remove(op.join(base_path, 'EEG.zip'))
break
assert op.isfile(op.join(datapath, 'cnt.mat')
), op.join(datapath, 'cnt.mat')
return [op.join(datapath, fn) for fn in ['cnt.mat', 'mrk.mat']]
def test_compare_hnn_core(self, run_hnn_core_fixture, backend, n_jobs=1):
"""Test hnn-core does not break."""
# small snippet of data on data branch for now. To be deleted
# later. Data branch should have only commit so it does not
# pollute the history.
data_url = ('https://raw.githubusercontent.com/jonescompneurolab/'
'hnn-core/test_data/dpl.txt')
if not op.exists('dpl.txt'):
_fetch_file(data_url, 'dpl.txt')
dpl_master = loadtxt('dpl.txt')
dpls, net = run_hnn_core_fixture(backend=backend)
dpl = dpls[0]
# write the dipole to a file and compare
fname = './dpl2.txt'
dpl.write(fname)
dpl_pr = loadtxt(fname)
assert_array_equal(dpl_pr[:, 2], dpl_master[:, 2]) # L2
assert_array_equal(dpl_pr[:, 3], dpl_master[:, 3]) # L5
# Test spike type counts
spike_type_counts = {}
for spike_gid in net.cell_response.spike_gids[0]:
if net.gid_to_type(spike_gid) not in spike_type_counts:
spike_type_counts[net.gid_to_type(spike_gid)] = 0
else:
spike_type_counts[net.gid_to_type(spike_gid)] += 1
assert 'common' not in spike_type_counts
assert 'exgauss' not in spike_type_counts
assert 'extpois' not in spike_type_counts
assert spike_type_counts == {
'evprox1': 269,
'L2_basket': 54,
'L2_pyramidal': 113,
'L5_pyramidal': 395,
'L5_basket': 85,
'evdist1': 234,
'evprox2': 269
}
def test_fetch_file():
"""Test file downloading
"""
# Skipping test if no internet connection available
try:
urllib.request.urlopen("http://github.com", timeout=2)
except urllib.request.URLError:
from nose.plugins.skip import SkipTest
raise SkipTest('No internet connection, skipping download test.')
urls = ['http://github.com/mne-tools/mne-python/blob/master/README.rst',
'ftp://surfer.nmr.mgh.harvard.edu/pub/data/bert.recon.md5sum.txt']
for url in urls:
archive_name = op.join(tempdir, "download_test")
_fetch_file(url, archive_name, print_destination=False)
assert_raises(Exception, _fetch_file, 'NOT_AN_ADDRESS',
op.join(tempdir, 'test'))
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, print_destination=False, resume=True)
def download_sample_data(dataset="ssvep", subject=1, session=1):
"""Download BCI data for example purpose
Parameters
----------
dataset : str
type of the dataset, could be "ssvep", "p300" or "imagery"
Default is "ssvep", as other are not implemented
subject : int
Subject id, dataset specific (default: 1)
session : int, default 1
Session number%load , dataset specific (default: 1)
Returns
-------
destination : str
Path to downloaded data
"""
if dataset == "ssvep":
DATASET_URL = 'https://zenodo.org/record/2392979/files/'
url = '{:s}subject{:02d}_run{:d}_raw.fif'.format(DATASET_URL,
subject, session + 1)
sign = 'SSVEPEXO'
key, key_dest = 'MNE_DATASETS_SSVEPEXO_PATH', 'MNE-ssvepexo-data'
elif dataset == "p300" or dataset == "imagery":
raise NotImplementedError("Not yet implemented")
# Use MNE _fetch_file to download EEG file
if get_config(key) is None:
set_config(key, os.path.join(os.path.expanduser("~"), "mne_data"))
# Adquire local de dados do MNE
path = _get_path(None, key, sign)
# Baixa o que está no URL para pasta local, que é criada no segundo parametro
destination = _url_to_local_path(url, os.path.join(path, key_dest))
# Cria pasta com todo caminho pro arquivo
os.makedirs(os.path.dirname(destination), exist_ok=True)
if not os.path.exists(destination):
_fetch_file(url, destination, print_destination=False)
return destination
def test_base_params():
"""Test params object with base params"""
param_url = ('https://raw.githubusercontent.com/jonescompneurolab/'
'hnn-core/test_data/base.json')
params_base_fname = op.join(hnn_core_root, 'param', 'base.json')
if not op.exists(params_base_fname):
_fetch_file(param_url, params_base_fname)
params_base = read_params(params_base_fname)
params = Params()
assert params == params_base
# unsupported extension
pytest.raises(ValueError, read_params, 'params.txt')
# empty file
empty_fname = op.join(hnn_core_root, 'param', 'empty.json')
with open(empty_fname, 'w') as json_data:
json.dump({}, json_data)
pytest.raises(ValueError, read_params, empty_fname)
# non dict type
pytest.raises(ValueError, Params, [])
pytest.raises(ValueError, Params, 'sdfdfdf')
def test_fetch_file():
"""Test file downloading
"""
tempdir = _TempDir()
urls = ['http://google.com', 'ftp://ftp.openbsd.org/pub/OpenBSD/README']
with ArgvSetter(disable_stderr=False): # to capture stdout
for url in urls:
archive_name = op.join(tempdir, "download_test")
_fetch_file(url,
archive_name,
timeout=30.,
verbose=False,
resume=False)
assert_raises(Exception,
_fetch_file,
'NOT_AN_ADDRESS',
op.join(tempdir, 'test'),
verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url,
resume_name,
resume=True,
timeout=30.,
verbose=False)
assert_raises(ValueError,
_fetch_file,
url,
archive_name,
hash_='a',
verbose=False)
assert_raises(RuntimeError,
_fetch_file,
url,
archive_name,
hash_='a' * 32,
verbose=False)
def test_fetch_file():
"""Test file downloading
"""
tempdir = _TempDir()
urls = ['http://google.com',
'ftp://ftp.openbsd.org/pub/OpenBSD/README']
with ArgvSetter(disable_stderr=False): # to capture stdout
for url in urls:
archive_name = op.join(tempdir, "download_test")
_fetch_file(url, archive_name, timeout=30., verbose=False)
assert_raises(Exception, _fetch_file, 'NOT_AN_ADDRESS',
op.join(tempdir, 'test'), verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, resume=True, timeout=30.,
verbose=False)
assert_raises(ValueError, _fetch_file, url, archive_name,
hash_='a', verbose=False)
assert_raises(RuntimeError, _fetch_file, url, archive_name,
hash_='a' * 32, verbose=False)
def test_read_legacy_params():
"""Test reading of legacy .param file."""
param_url = ('https://raw.githubusercontent.com/hnnsolver/'
'hnn-core/test_data/default.param')
params_legacy_fname = op.join(hnn_core_root, 'param', 'default.param')
if not op.exists(params_legacy_fname):
_fetch_file(param_url, params_legacy_fname)
params_new_fname = op.join(hnn_core_root, 'param', 'default.json')
params_legacy = read_params(params_legacy_fname)
params_new = read_params(params_new_fname)
params_new_seedless = {
key: val
for key, val in params_new.items()
if key not in params_new['prng_seedcore*'].keys()
}
params_legacy_seedless = {
key: val
for key, val in params_legacy.items()
if key not in params_legacy['prng_seedcore*'].keys()
}
assert params_new_seedless == params_legacy_seedless
def eeg_data_path(base_path, subject):
datapath = op.join(base_path, "EEG", "subject {:02d}".format(subject),
"with occular artifact")
if not op.isfile(op.join(datapath, "cnt.mat")):
if not op.isdir(op.join(base_path, "EEG")):
os.makedirs(op.join(base_path, "EEG"))
intervals = [[1, 5], [6, 10], [11, 15], [16, 20], [21, 25], [26, 29]]
for low, high in intervals:
if subject >= low and subject <= high:
if not op.isfile(op.join(base_path, "EEG.zip")):
_fetch_file(
"{}/EEG/EEG_{:02d}-{:02d}.zip".format(
SHIN_URL, low, high),
op.join(base_path, "EEG.zip"),
print_destination=False,
)
with z.ZipFile(op.join(base_path, "EEG.zip"), "r") as f:
f.extractall(op.join(base_path, "EEG"))
os.remove(op.join(base_path, "EEG.zip"))
break
assert op.isfile(op.join(datapath,
"cnt.mat")), op.join(datapath, "cnt.mat")
return [op.join(datapath, fn) for fn in ["cnt.mat", "mrk.mat"]]
def test_fetch_file():
"""Test file downloading
"""
# Skipping test if no internet connection available
try:
urllib.request.urlopen("http://github.com", timeout=2)
except:
from nose.plugins.skip import SkipTest
raise SkipTest('No internet connection, skipping download test.')
urls = [
'http://github.com/mne-tools/mne-python/blob/master/README.rst',
'ftp://surfer.nmr.mgh.harvard.edu/pub/data/bert.recon.md5sum.txt'
]
for url in urls:
archive_name = op.join(tempdir, "download_test")
_fetch_file(url, archive_name, print_destination=False)
assert_raises(Exception, _fetch_file, 'NOT_AN_ADDRESS',
op.join(tempdir, 'test'))
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, print_destination=False, resume=True)
def local_data_path(base_path, subject):
if not os.path.isdir(os.path.join(base_path,
'subject_{}'.format(subject))):
if not os.path.isdir(os.path.join(base_path, 'data')):
_fetch_file(DATA_PATH, os.path.join(base_path, 'data.zip'),
print_destination=False)
with z.ZipFile(os.path.join(base_path, 'data.zip'), 'r') as f:
f.extractall(base_path)
os.remove(os.path.join(base_path, 'data.zip'))
datapath = os.path.join(base_path, 'data')
for i in range(1, 5):
os.makedirs(os.path.join(base_path, 'subject_{}'.format(i)))
for session in range(1, 4):
for run in ['A', 'B']:
os.rename(os.path.join(datapath,
'S{}_{}{}.cnt'.format(i, session,
run)),
os.path.join(base_path,
'subject_{}'.format(i),
'{}{}.cnt'.format(session, run)))
shutil.rmtree(os.path.join(base_path, 'data'))
subjpath = os.path.join(base_path, 'subject_{}'.format(subject))
return [[os.path.join(subjpath, '{}{}.cnt'.format(y, x))
for x in ['A', 'B']] for y in ['1', '2', '3']]
def test_constants(tmpdir):
"""Test compensation."""
tmpdir = str(tmpdir) # old pytest...
dest = op.join(tmpdir, 'fiff.zip')
_fetch_file('https://codeload.github.com/mne-tools/fiff-constants/zip/' +
commit, dest)
names = list()
with zipfile.ZipFile(dest, 'r') as ff:
for name in ff.namelist():
if 'Dictionary' in name:
ff.extract(name, tmpdir)
names.append(op.basename(name))
shutil.move(op.join(tmpdir, name), op.join(tmpdir, names[-1]))
names = sorted(names)
assert names == ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt',
'DictionaryStructures.txt',
'DictionaryTags.txt', 'DictionaryTags_MNE.txt',
'DictionaryTypes.txt', 'DictionaryTypes_MNE.txt']
# IOD (MEGIN and MNE)
fif = dict(iod=dict(), tags=dict(), types=dict(), defines=dict())
con = dict(iod=dict(), tags=dict(), types=dict(), defines=dict())
fiff_version = None
for name in ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt']:
with open(op.join(tmpdir, name), 'rb') as fid:
for line in fid:
line = line.decode('latin1').strip()
if line.startswith('# Packing revision'):
assert fiff_version is None
fiff_version = line.split()[-1]
if (line.startswith('#') or line.startswith('alias') or
len(line) == 0):
continue
line = line.split('"')
assert len(line) in (1, 2, 3)
desc = '' if len(line) == 1 else line[1]
line = line[0].split()
assert len(line) in (2, 3)
if len(line) == 2:
kind, id_ = line
else:
kind, id_, tagged = line
assert tagged in ('tagged',)
id_ = int(id_)
if id_ not in iod_dups:
assert id_ not in fif['iod']
fif['iod'][id_] = [kind, desc]
# Tags (MEGIN)
with open(op.join(tmpdir, 'DictionaryTags.txt'), 'rb') as fid:
for line in fid:
line = line.decode('ISO-8859-1').strip()
if (line.startswith('#') or line.startswith('alias') or
line.startswith(':') or len(line) == 0):
continue
line = line.split('"')
assert len(line) in (1, 2, 3), line
desc = '' if len(line) == 1 else line[1]
line = line[0].split()
assert len(line) == 4, line
kind, id_, dtype, unit = line
id_ = int(id_)
val = [kind, dtype, unit]
assert id_ not in fif['tags'], (fif['tags'].get(id_), val)
fif['tags'][id_] = val
# Tags (MNE)
with open(op.join(tmpdir, 'DictionaryTags_MNE.txt'), 'rb') as fid:
for li, line in enumerate(fid):
line = line.decode('ISO-8859-1').strip()
# ignore continuation lines (*)
if (line.startswith('#') or line.startswith('alias') or
line.startswith(':') or line.startswith('*') or
len(line) == 0):
continue
# weird syntax around line 80:
if line in ('/*', '"'):
continue
line = line.split('"')
assert len(line) in (1, 2, 3), line
if len(line) == 3 and len(line[2]) > 0:
l2 = line[2].strip()
assert l2.startswith('/*') and l2.endswith('*/'), l2
desc = '' if len(line) == 1 else line[1]
line = line[0].split()
assert len(line) == 3, (li + 1, line)
kind, id_, dtype = line
unit = '-'
id_ = int(id_)
val = [kind, dtype, unit]
if id_ not in tag_dups:
assert id_ not in fif['tags'], (fif['tags'].get(id_), val)
fif['tags'][id_] = val
# Types and enums
in_ = None
re_prim = re.compile(r'^primitive\((.*)\)\s*(\S*)\s*"(.*)"$')
re_enum = re.compile(r'^enum\((\S*)\)\s*".*"$')
re_enum_entry = re.compile(r'\s*(\S*)\s*(\S*)\s*"(.*)"$')
re_defi = re.compile(r'#define\s*(\S*)\s*(\S*)\s*"(.*)"$')
used_enums = list()
for extra in ('', '_MNE'):
with open(op.join(tmpdir, 'DictionaryTypes%s.txt'
% (extra,)), 'rb') as fid:
for li, line in enumerate(fid):
line = line.decode('ISO-8859-1').strip()
if in_ is None:
p = re_prim.match(line)
e = re_enum.match(line)
d = re_defi.match(line)
if p is not None:
t, s, d = p.groups()
s = int(s)
assert s not in fif['types']
fif['types'][s] = [t, d]
elif e is not None:
# entering an enum
this_enum = e.group(1)
if this_enum not in fif:
used_enums.append(this_enum)
fif[this_enum] = dict()
con[this_enum] = dict()
in_ = fif[this_enum]
elif d is not None:
t, s, d = d.groups()
s = int(s)
fif['defines'][t] = [s, d]
else:
assert not line.startswith('enum(')
else: # in an enum
if line == '{':
continue
elif line == '}':
in_ = None
continue
t, s, d = re_enum_entry.match(line).groups()
s = int(s)
if t != 'ecg' and s != 3: # ecg defined the same way
assert s not in in_
in_[s] = [t, d]
#
# Assertions
#
# Version
mne_version = '%d.%d' % (FIFF.FIFFC_MAJOR_VERSION,
FIFF.FIFFC_MINOR_VERSION)
assert fiff_version == mne_version
unknowns = list()
# Assert that all our constants are in the FIF def
assert 'FIFFV_SSS_JOB_NOTHING' in dir(FIFF)
for name in sorted(dir(FIFF)):
if name.startswith('_') or name in _dir_ignore_names:
continue
check = None
val = getattr(FIFF, name)
if name in fif['defines']:
assert fif['defines'][name][0] == val
elif name.startswith('FIFFC_'):
# Checked above
assert name in ('FIFFC_MAJOR_VERSION', 'FIFFC_MINOR_VERSION',
'FIFFC_VERSION')
elif name.startswith('FIFFB_'):
check = 'iod'
elif name.startswith('FIFFT_'):
check = 'types'
elif name.startswith('FIFFV_'):
if name.startswith('FIFFV_MNE_') and name.endswith('_ORI'):
check = 'mne_ori'
elif name.startswith('FIFFV_MNE_') and name.endswith('_COV'):
check = 'covariance_type'
elif name.startswith('FIFFV_MNE_COORD'):
check = 'coord' # weird wrapper
elif name.endswith('_CH') or '_QUAT_' in name or name in \
('FIFFV_DIPOLE_WAVE', 'FIFFV_GOODNESS_FIT',
'FIFFV_HPI_ERR', 'FIFFV_HPI_G', 'FIFFV_HPI_MOV'):
check = 'ch_type'
elif name.startswith('FIFFV_SUBJ_'):
check = name.split('_')[2].lower()
elif name in ('FIFFV_POINT_LPA', 'FIFFV_POINT_NASION',
'FIFFV_POINT_RPA'):
check = 'cardinal_point'
else:
for check in used_enums:
if name.startswith('FIFFV_' + check.upper()):
break
else:
raise RuntimeError('Could not find %s' % (name,))
assert check in used_enums, name
if 'SSS' in check:
raise RuntimeError
elif name.startswith('FIFF_UNIT'): # units and multipliers
check = name.split('_')[1].lower()
elif name.startswith('FIFF_'):
check = 'tags'
else:
unknowns.append((name, val))
if check is not None and name not in _tag_ignore_names:
assert val in fif[check], '%s: %s, %s' % (check, val, name)
if val in con[check]:
msg = "%s='%s' ?" % (name, con[check][val])
assert _aliases.get(name) == con[check][val], msg
else:
con[check][val] = name
unknowns = '\n\t'.join('%s (%s)' % u for u in unknowns)
assert len(unknowns) == 0, 'Unknown types\n\t%s' % unknowns
# Assert that all the FIF defs are in our constants
assert set(fif.keys()) == set(con.keys())
for key in sorted(set(fif.keys()) - {'defines'}):
this_fif, this_con = fif[key], con[key]
assert len(set(this_fif.keys())) == len(this_fif)
assert len(set(this_con.keys())) == len(this_con)
missing_from_con = sorted(set(this_con.keys()) - set(this_fif.keys()))
assert missing_from_con == [], key
if key not in _ignore_incomplete_enums:
missing_from_fif = sorted(set(this_fif.keys()) -
set(this_con.keys()))
assert missing_from_fif == [], key
# Assert that `coil_def.dat` has accurate descriptions of all enum(coil)
coil_def = _read_coil_defs()
coil_desc = np.array([c['desc'] for c in coil_def])
coil_def = np.array([(c['coil_type'], c['accuracy'])
for c in coil_def], int)
mask = (coil_def[:, 1] == FWD.COIL_ACCURACY_ACCURATE)
coil_def = coil_def[mask, 0]
coil_desc = coil_desc[mask]
bad_list = []
for key in fif['coil']:
if key not in _missing_coil_def and key not in coil_def:
bad_list.append((' %s,' % key).ljust(10) +
' # ' + fif['coil'][key][1])
assert len(bad_list) == 0, '\n' + '\n'.join(bad_list)
# Assert that enum(coil) has all `coil_def.dat` entries
for key, desc in zip(coil_def, coil_desc):
if key not in fif['coil']:
bad_list.append((' %s,' % key).ljust(10) + ' # ' + desc)
assert len(bad_list) == 0, '\n' + '\n'.join(bad_list)
def test_fetch_file(url, tmpdir):
"""Test URL retrieval."""
tempdir = str(tmpdir)
archive_name = op.join(tempdir, "download_test")
with catch_logging() as log:
_fetch_file(url, archive_name, timeout=30., verbose='debug')
log = log.getvalue()
assert 'Resuming at' not in log
with open(archive_name, 'rb') as fid:
data = fid.read()
stop = len(data) // 2
assert 0 < stop < len(data)
with open(archive_name + '.part', 'wb') as fid:
fid.write(data[:stop])
with catch_logging() as log:
_fetch_file(url, archive_name, timeout=30., verbose='debug')
log = log.getvalue()
assert 'Resuming at %s' % stop in log
with pytest.raises(Exception, match='unknown url type'):
_fetch_file('NOT_AN_ADDRESS', op.join(tempdir, 'test'), verbose=False)
resume_name = op.join(tempdir, "download_resume")
# touch file
with open(resume_name + '.part', 'w'):
os.utime(resume_name + '.part', None)
_fetch_file(url, resume_name, resume=True, timeout=30.,
verbose=False)
with pytest.raises(ValueError, match='Bad hash value'):
_fetch_file(url, archive_name, hash_='a', verbose=False)
with pytest.raises(RuntimeError, match='Hash mismatch'):
_fetch_file(url, archive_name, hash_='a' * 32, verbose=False)
def test_constants(tmpdir):
"""Test compensation."""
tmpdir = str(tmpdir) # old pytest...
dest = op.join(tmpdir, 'fiff.zip')
_fetch_file('https://codeload.github.com/mne-tools/fiff-constants/zip/'
'master', dest)
names = list()
with zipfile.ZipFile(dest, 'r') as ff:
for name in ff.namelist():
if 'Dictionary' in name:
ff.extract(name, tmpdir)
names.append(op.basename(name))
shutil.move(op.join(tmpdir, name), op.join(tmpdir, names[-1]))
names = sorted(names)
assert names == ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt',
'DictionaryStructures.txt',
'DictionaryTags.txt', 'DictionaryTags_MNE.txt',
'DictionaryTypes.txt', 'DictionaryTypes_MNE.txt']
# IOD (MEGIN and MNE)
iod = dict()
fiff_version = None
for name in ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt']:
with open(op.join(tmpdir, name), 'rb') as fid:
for line in fid:
line = line.decode('latin1').strip()
if line.startswith('# Packing revision'):
assert fiff_version is None
fiff_version = line.split()[-1]
if (line.startswith('#') or line.startswith('alias') or
len(line) == 0):
continue
line = line.split('"')
assert len(line) in (1, 2, 3)
desc = '' if len(line) == 1 else line[1]
line = line[0].split()
assert len(line) in (2, 3)
if len(line) == 2:
kind, id_ = line
else:
kind, id_, tagged = line
assert tagged in ('tagged',)
id_ = int(id_)
if id_ not in iod_dups:
assert id_ not in iod
iod[id_] = [kind, desc]
# Tags (MEGIN)
tags = dict()
with open(op.join(tmpdir, 'DictionaryTags.txt'), 'rb') as fid:
for line in fid:
line = line.decode('ISO-8859-1').strip()
if (line.startswith('#') or line.startswith('alias') or
line.startswith(':') or len(line) == 0):
continue
line = line.split('"')
assert len(line) in (1, 2, 3), line
desc = '' if len(line) == 1 else line[1]
line = line[0].split()
assert len(line) == 4, line
kind, id_, dtype, unit = line
id_ = int(id_)
val = [kind, dtype, unit]
assert id_ not in tags, (tags.get(id_), val)
tags[id_] = val
# Tags (MNE)
with open(op.join(tmpdir, 'DictionaryTags_MNE.txt'), 'rb') as fid:
for li, line in enumerate(fid):
line = line.decode('ISO-8859-1').strip()
# ignore continuation lines (*)
if (line.startswith('#') or line.startswith('alias') or
line.startswith(':') or line.startswith('*') or
len(line) == 0):
continue
# weird syntax around line 80:
if line in ('/*', '"'):
continue
line = line.split('"')
assert len(line) in (1, 2, 3), line
if len(line) == 3 and len(line[2]) > 0:
l2 = line[2].strip()
assert l2.startswith('/*') and l2.endswith('*/'), l2
desc = '' if len(line) == 1 else line[1]
line = line[0].split()
assert len(line) == 3, (li + 1, line)
kind, id_, dtype = line
unit = '-'
id_ = int(id_)
val = [kind, dtype, unit]
if id_ not in tag_dups:
assert id_ not in tags, (tags.get(id_), val)
tags[id_] = val
# Types and enums
defines = dict() # maps the other way (name->val)
types = dict()
used_enums = ('unit', 'unitm', 'coil', 'aspect', 'bem_surf_id',
'ch_type', 'coord', 'mri_pixel', 'point', 'role',
'hand', 'sex', 'proj_item', 'bem_approx',
'mne_cov_ch', 'mne_ori', 'mne_map', 'covariance_type',
'mne_priors', 'mne_space', 'mne_surf')
enums = dict((k, dict()) for k in used_enums)
in_ = None
re_prim = re.compile(r'^primitive\((.*)\)\s*(\S*)\s*"(.*)"$')
re_enum = re.compile(r'^enum\((\S*)\)\s*".*"$')
re_enum_entry = re.compile(r'\s*(\S*)\s*(\S*)\s*"(.*)"$')
re_defi = re.compile(r'#define\s*(\S*)\s*(\S*)\s*"(.*)"$')
for extra in ('', '_MNE'):
with open(op.join(tmpdir, 'DictionaryTypes%s.txt'
% (extra,)), 'rb') as fid:
for li, line in enumerate(fid):
line = line.decode('ISO-8859-1').strip()
if in_ is None:
p = re_prim.match(line)
e = re_enum.match(line)
d = re_defi.match(line)
if p is not None:
t, s, d = p.groups()
s = int(s)
assert s not in types
types[s] = [t, d]
elif e is not None:
# entering an enum
this_enum = e.group(1)
if this_enum in enums:
in_ = enums[e.group(1)]
elif d is not None:
t, s, d = d.groups()
s = int(s)
defines[t] = [s, d]
else:
assert not line.startswith('enum(')
else: # in an enum
if line == '{':
continue
elif line == '}':
in_ = None
continue
t, s, d = re_enum_entry.match(line).groups()
s = int(s)
if t != 'ecg' and s != 3: # ecg defined the same way
assert s not in in_
in_[s] = [t, d]
#
# Assertions
#
# Version
mne_version = '%d.%d' % (FIFF.FIFFC_MAJOR_VERSION,
FIFF.FIFFC_MINOR_VERSION)
assert fiff_version == mne_version
unknowns = list()
# Assert that all our constants are in the dict
# (we are not necessarily complete the other way)
for name in sorted(dir(FIFF)):
if name.startswith('_') or name in _dir_ignore_names or \
name in _missing_names:
continue
val = getattr(FIFF, name)
if name in defines:
assert defines[name][0] == val
elif name.startswith('FIFFC_'):
# Checked above
assert name in ('FIFFC_MAJOR_VERSION', 'FIFFC_MINOR_VERSION',
'FIFFC_VERSION')
elif name.startswith('FIFFB_'):
assert val in iod, (val, name)
elif name.startswith('FIFFT_'):
assert val in types, (val, name)
elif name.startswith('FIFFV_'):
if name.startswith('FIFFV_MNE_') and name.endswith('_ORI'):
this_enum = 'mne_ori'
elif name.startswith('FIFFV_MNE_') and name.endswith('_COV'):
this_enum = 'covariance_type'
elif name.startswith('FIFFV_MNE_COORD'):
this_enum = 'coord' # weird wrapper
elif name.endswith('_CH') or '_QUAT_' in name or name in \
('FIFFV_DIPOLE_WAVE', 'FIFFV_GOODNESS_FIT',
'FIFFV_HPI_ERR', 'FIFFV_HPI_G', 'FIFFV_HPI_MOV'):
this_enum = 'ch_type'
elif name.startswith('FIFFV_SUBJ_'):
this_enum = name.split('_')[2].lower()
else:
for this_enum in used_enums:
if name.startswith('FIFFV_' + this_enum.upper()):
break
else:
raise RuntimeError('Could not find %s' % (name,))
assert this_enum in used_enums, name
assert val in enums[this_enum], (val, name)
elif name.startswith('FIFF_UNIT'): # units and multipliers
this_enum = name.split('_')[1].lower()
assert val in enums[this_enum], (name, val)
elif name.startswith('FIFF_'):
assert val in tags, (name, val)
else:
unknowns.append((name, val))
unknowns = '\n\t'.join('%s (%s)' % u for u in unknowns)
assert len(unknowns) == 0, 'Unknown types\n\t%s' % unknowns
