def test_get_matched_emptyroom_no_meas_date():
"""Test that we warn if measurement date can be read or inferred."""
bids_root = _TempDir()
er_session = 'mysession'
er_meas_date = None
er_dir = make_bids_folders(subject='emptyroom',
session=er_session,
kind='meg',
bids_root=bids_root)
er_bids_path = BIDSPath(subject='emptyroom',
session=er_session,
task='noise')
er_basename = str(er_bids_path)
raw = mne.io.read_raw_fif(raw_fname)
er_raw_fname = op.join(data_path, 'MEG', 'sample', 'ernoise_raw.fif')
raw.copy().crop(0, 10).save(er_raw_fname, overwrite=True)
er_raw = mne.io.read_raw_fif(er_raw_fname)
er_raw.set_meas_date(er_meas_date)
er_raw.save(op.join(er_dir, f'{er_basename}_meg.fif'), overwrite=True)
# Write raw file data using mne-bids, and remove participants.tsv
# as it's incomplete (doesn't contain the emptyroom subject we wrote
# manually using MNE's Raw.save() above)
raw = mne.io.read_raw_fif(raw_fname)
write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
os.remove(op.join(bids_root, 'participants.tsv'))
with pytest.warns(RuntimeWarning, match='Could not retrieve .* date'):
get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_root)
def test_get_matched_emptyroom_ties():
"""Test that we receive a warning on a date tie."""
bids_root = _TempDir()
session = '20010101'
er_dir = make_bids_folders(subject='emptyroom',
session=session,
kind='meg',
bids_root=bids_root)
meas_date = (datetime.strptime(session,
'%Y%m%d').replace(tzinfo=timezone.utc))
raw = mne.io.read_raw_fif(raw_fname)
er_raw_fname = op.join(data_path, 'MEG', 'sample', 'ernoise_raw.fif')
raw.copy().crop(0, 10).save(er_raw_fname, overwrite=True)
er_raw = mne.io.read_raw_fif(er_raw_fname)
if check_version('mne', '0.20'):
raw.set_meas_date(meas_date)
er_raw.set_meas_date(meas_date)
else:
raw.info['meas_date'] = (meas_date.timestamp(), 0)
er_raw.info['meas_date'] = (meas_date.timestamp(), 0)
write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
er_bids_path = BIDSPath(subject='emptyroom', session=session)
er_basename_1 = str(er_bids_path)
er_basename_2 = make_bids_basename(subject='emptyroom',
session=session,
task='noise')
er_raw.save(op.join(er_dir, f'{er_basename_1}_meg.fif'))
er_raw.save(op.join(er_dir, f'{er_basename_2}_meg.fif'))
with pytest.warns(RuntimeWarning, match='Found more than one'):
get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_root)
def test_get_matched_empty_room():
"""Test reading of empty room data."""
bids_root = _TempDir()
raw = mne.io.read_raw_fif(raw_fname)
bids_basename = make_bids_basename(subject='01',
session='01',
task='audiovisual',
run='01')
write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
er_basename = get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_root)
assert er_basename is None
# testing data has no noise recording, so save the actual data
# as if it were noise
er_raw_fname = op.join(data_path, 'MEG', 'sample', 'ernoise_raw.fif')
raw.crop(0, 10).save(er_raw_fname, overwrite=True)
er_raw = mne.io.read_raw_fif(er_raw_fname)
er_date = er_raw.info['meas_date']
if not isinstance(er_date, datetime):
# mne < v0.20
er_date = datetime.fromtimestamp(er_raw.info['meas_date'][0])
er_date = er_date.strftime('%Y%m%d')
er_bids_basename = make_bids_basename(subject='emptyroom',
task='noise',
session=er_date)
write_raw_bids(er_raw, er_bids_basename, bids_root, overwrite=True)
recovered_er_basename = get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_root)
assert er_bids_basename == recovered_er_basename
# assert that we get best emptyroom if there are multiple available
sh.rmtree(op.join(bids_root, 'sub-emptyroom'))
dates = ['20021204', '20021201', '20021001']
for date in dates:
er_bids_basename.update(session=date)
er_meas_date = datetime.strptime(date, '%Y%m%d')
er_meas_date = er_meas_date.replace(tzinfo=timezone.utc)
if check_version('mne', '0.20'):
er_raw.set_meas_date(er_meas_date)
else:
er_raw.info['meas_date'] = (er_meas_date.timestamp(), 0)
write_raw_bids(er_raw, er_bids_basename, bids_root)
best_er_basename = get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_root)
assert '20021204' in best_er_basename
# assert that we get error if meas_date is not available.
raw = read_raw_bids(bids_basename=bids_basename,
bids_root=bids_root,
kind='meg')
if check_version('mne', '0.20'):
raw.set_meas_date(None)
else:
raw.info['meas_date'] = None
raw.annotations.orig_time = None
anonymize_info(raw.info)
write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
with pytest.raises(ValueError,
match='The provided recording does not '
'have a measurement date set'):
get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_root)
def run_maxwell_filter(subject, session=None):
deriv_path = config.get_subject_deriv_path(subject=subject,
session=session,
kind=config.get_kind())
os.makedirs(deriv_path, exist_ok=True)
for run_idx, run in enumerate(config.get_runs()):
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=run,
processing=config.proc,
recording=config.rec,
space=config.space)
raw = load_data(bids_basename)
if run_idx == 0:
dev_head_t = raw.info['dev_head_t'] # Re-use in all runs.
# Auto-detect bad channels.
if config.find_flat_channels_meg or config.find_noisy_channels_meg:
find_bad_channels(raw=raw,
subject=subject,
session=session,
task=config.get_task(),
run=run)
# Maxwell-filter experimental data.
if config.use_maxwell_filter:
msg = 'Applying Maxwell filter to experimental data.'
logger.info(
gen_log_message(message=msg,
step=1,
subject=subject,
session=session))
# Warn if no bad channels are set before Maxwell filter
if not raw.info['bads']:
msg = '\nFound no bad channels. \n '
logger.warn(
gen_log_message(message=msg,
subject=subject,
step=1,
session=session))
if config.mf_st_duration:
msg = ' st_duration=%d' % (config.mf_st_duration)
logger.info(
gen_log_message(message=msg,
step=1,
subject=subject,
session=session))
# Keyword arguments shared between Maxwell filtering of the
# experimental and the empty-room data.
common_mf_kws = dict(calibration=config.mf_cal_fname,
cross_talk=config.mf_ctc_fname,
st_duration=config.mf_st_duration,
origin=config.mf_head_origin,
coord_frame='head',
destination=dev_head_t)
raw_sss = mne.preprocessing.maxwell_filter(raw, **common_mf_kws)
raw_out = raw_sss
raw_fname_out = op.join(deriv_path, f'{bids_basename}_sss_raw.fif')
else:
msg = ('Not applying Maxwell filter.\nIf you wish to apply it, '
'set use_maxwell_filter=True in your configuration.')
logger.info(
gen_log_message(message=msg,
step=1,
subject=subject,
session=session))
raw_out = raw
raw_fname_out = op.join(deriv_path,
f'{bids_basename}_nosss_raw.fif')
raw_out.save(raw_fname_out, overwrite=True)
if config.interactive:
raw_out.plot(n_channels=50, butterfly=True)
# Empty-room processing.
#
# We pick the empty-room recording closest in time to the first run
# of the experimental session.
if run_idx == 0 and config.noise_cov == 'emptyroom':
msg = 'Processing empty-room recording …'
logger.info(
gen_log_message(step=1,
subject=subject,
session=session,
message=msg))
bids_basename_er_in = get_matched_empty_room(
bids_basename=bids_basename, bids_root=config.bids_root)
raw_er = load_data(bids_basename_er_in)
raw_er.info['bads'] = [
ch for ch in raw.info['bads'] if ch.startswith('MEG')
]
# Maxwell-filter empty-room data.
if config.use_maxwell_filter:
msg = 'Applying Maxwell filter to empty-room recording'
logger.info(
gen_log_message(message=msg,
step=1,
subject=subject,
session=session))
# We want to ensure we use the same coordinate frame origin in
# empty-room and experimental data processing. To do this, we
# inject the sensor locations and the head <> device transform
# into the empty-room recording's info, and leave all other
# parameters the same as for the experimental data. This is not
# very clean, as we normally should not alter info manually,
# except for info['bads']. Will need improvement upstream in
# MNE-Python.
raw_er.info['dig'] = raw.info['dig']
raw_er.info['dev_head_t'] = dev_head_t
raw_er_sss = mne.preprocessing.maxwell_filter(
raw_er, **common_mf_kws)
# Perform a sanity check: empty-room rank should match the
# experimental data rank after Maxwell filtering.
rank_exp = mne.compute_rank(raw, rank='info')['meg']
rank_er = mne.compute_rank(raw_er, rank='info')['meg']
if not np.isclose(rank_exp, rank_er):
msg = (f'Experimental data rank {rank_exp:.1f} does not '
f'match empty-room data rank {rank_er:.1f} after '
f'Maxwell filtering. This indicates that the data '
f'were processed differenlty.')
raise RuntimeError(msg)
raw_er_out = raw_er_sss
raw_er_fname_out = op.join(
deriv_path, f'{bids_basename}_emptyroom_sss_raw.fif')
else:
raw_er_out = raw_er
raw_er_fname_out = op.join(
deriv_path, f'{bids_basename}_emptyroom_nosss_raw.fif')
raw_er_out.save(raw_er_fname_out, overwrite=True)
# will be different on different days.
dates = ['20021204', '20021201', '20021001']
for date in dates:
er_bids_basename = make_bids_basename(subject='emptyroom', session=date,
task='noise')
er_meas_date = datetime.strptime(date, '%Y%m%d')
er_raw.set_meas_date(er_meas_date.replace(tzinfo=timezone.utc))
write_raw_bids(er_raw, er_bids_basename, bids_path, overwrite=True)
###############################################################################
# Let us look at the directory structure
from mne_bids.utils import print_dir_tree # noqa
print_dir_tree(bids_path)
###############################################################################
# To get an accurate estimate of the noise, it is important that the empty
# room recording be as close in date as the raw data.
# We can retrieve the filename corresponding to the empty room
# file that is closest in time to the measurement file using MNE-BIDS.
from mne_bids import get_matched_empty_room # noqa
bids_fname = bids_basename + '_meg.fif'
best_er_fname = get_matched_empty_room(bids_fname, bids_path)
print(best_er_fname)
###############################################################################
# Finally, we can read the empty room file using
raw = read_raw_bids(best_er_fname, bids_path)
dates = ['20021204', '20021201', '20021001']
for date in dates:
er_bids_basename = make_bids_basename(subject='emptyroom',
session=date,
task='noise')
er_meas_date = datetime.strptime(date, '%Y%m%d')
er_raw.set_meas_date(er_meas_date.replace(tzinfo=timezone.utc))
write_raw_bids(er_raw, er_bids_basename, bids_path, overwrite=True)
###############################################################################
# Let us look at the directory structure
from mne_bids.utils import print_dir_tree # noqa
print_dir_tree(bids_path)
###############################################################################
# To get an accurate estimate of the noise, it is important that the empty
# room recording be as close in date as the raw data.
# We can retrieve the basename corresponding to the empty room
# recording that is closest in time to the experimental measurement.
from mne_bids import get_matched_empty_room # noqa
best_er_basename = get_matched_empty_room(bids_basename=bids_basename,
bids_root=bids_path)
print(best_er_basename)
###############################################################################
# Finally, we can read the empty room file using
raw = read_raw_bids(bids_basename=best_er_basename, bids_root=bids_path)
def run_maxwell_filter(subject, session=None):
deriv_path = config.get_subject_deriv_path(subject=subject,
session=session,
kind=config.get_kind())
os.makedirs(deriv_path, exist_ok=True)
if config.proc and 'sss' in config.proc and config.use_maxwell_filter:
raise ValueError(f'You cannot set use_maxwell_filter to True '
f'if data have already processed with Maxwell-filter.'
f' Got proc={config.proc}.')
# Load dev_head_t and digitization points from reference run.
# Re-use in all runs and for processing empty-room recording.
reference_run = config.get_mf_reference_run()
bids_basename = BIDSPath(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=reference_run,
processing=config.proc,
recording=config.rec,
space=config.space)
raw = load_data(bids_basename) # XXX Loading info would suffice!
dev_head_t = raw.info['dev_head_t']
dig = raw.info['dig']
del reference_run, raw, bids_basename
for run in config.get_runs():
bids_basename = BIDSPath(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=run,
processing=config.proc,
recording=config.rec,
space=config.space,
kind=config.get_kind())
raw = load_data(bids_basename)
# Auto-detect bad channels.
if config.find_flat_channels_meg or config.find_noisy_channels_meg:
find_bad_channels(raw=raw, subject=subject, session=session,
task=config.get_task(), run=run)
# Maxwell-filter experimental data.
if config.use_maxwell_filter:
msg = 'Applying Maxwell filter to experimental data.'
logger.info(gen_log_message(message=msg, step=1, subject=subject,
session=session))
# Warn if no bad channels are set before Maxwell filter
if not raw.info['bads']:
msg = '\nFound no bad channels. \n '
logger.warning(gen_log_message(message=msg, subject=subject,
step=1, session=session))
if config.mf_st_duration:
msg = ' st_duration=%d' % (config.mf_st_duration)
logger.info(gen_log_message(message=msg, step=1,
subject=subject, session=session))
# Keyword arguments shared between Maxwell filtering of the
# experimental and the empty-room data.
common_mf_kws = dict(calibration=config.mf_cal_fname,
cross_talk=config.mf_ctc_fname,
st_duration=config.mf_st_duration,
origin=config.mf_head_origin,
coord_frame='head',
destination=dev_head_t)
raw_sss = mne.preprocessing.maxwell_filter(raw, **common_mf_kws)
raw_out = raw_sss
raw_fname_out = (bids_basename.copy()
.update(prefix=deriv_path,
processing='sss',
extension='.fif'))
else:
msg = ('Not applying Maxwell filter.\nIf you wish to apply it, '
'set use_maxwell_filter=True in your configuration.')
logger.info(gen_log_message(message=msg, step=1, subject=subject,
session=session))
raw_out = raw
raw_fname_out = (bids_basename.copy()
.update(prefix=deriv_path,
extension='.fif'))
# Save only the channel types we wish to analyze.
# We do not rum `raw_out.pick()` here because it uses too much memory.
chs_to_include = config.get_picks(raw_out.info)
raw_out.save(raw_fname_out, picks=chs_to_include, overwrite=True)
del raw_out
if config.interactive:
# Load the data we have just written, because it contains only
# the relevant channels.
raw = mne.io.read_raw_fif(raw_fname_out, allow_maxshield=True)
raw.plot(n_channels=50, butterfly=True)
# Empty-room processing.
#
# We pick the empty-room recording closest in time to the first run
# of the experimental session.
if config.process_er:
msg = 'Processing empty-room recording …'
logger.info(gen_log_message(step=1, subject=subject,
session=session, message=msg))
bids_basename_er_in = get_matched_empty_room(
bids_basename=bids_basename,
bids_root=config.bids_root)
raw_er = load_data(bids_basename_er_in)
raw_er.info['bads'] = [ch for ch in raw.info['bads'] if
ch.startswith('MEG')]
# Maxwell-filter empty-room data.
if config.use_maxwell_filter:
msg = 'Applying Maxwell filter to empty-room recording'
logger.info(gen_log_message(message=msg, step=1,
subject=subject, session=session))
# We want to ensure we use the same coordinate frame origin in
# empty-room and experimental data processing. To do this, we
# inject the sensor locations and the head <> device transform
# into the empty-room recording's info, and leave all other
# parameters the same as for the experimental data. This is not
# very clean, as we normally should not alter info manually,
# except for info['bads']. Will need improvement upstream in
# MNE-Python.
raw_er.info['dig'] = dig
raw_er.info['dev_head_t'] = dev_head_t
raw_er_sss = mne.preprocessing.maxwell_filter(raw_er,
**common_mf_kws)
# Perform a sanity check: empty-room rank should match the
# experimental data rank after Maxwell filtering.
rank_exp = mne.compute_rank(raw, rank='info')['meg']
rank_er = mne.compute_rank(raw_er, rank='info')['meg']
if not np.isclose(rank_exp, rank_er):
msg = (f'Experimental data rank {rank_exp:.1f} does not '
f'match empty-room data rank {rank_er:.1f} after '
f'Maxwell filtering. This indicates that the data '
f'were processed differenlty.')
raise RuntimeError(msg)
raw_er_out = raw_er_sss
raw_er_fname_out = bids_basename.copy().update(
processing='sss')
else:
raw_er_out = raw_er
raw_er_fname_out = bids_basename.copy()
raw_er_fname_out = raw_er_fname_out.update(prefix=deriv_path,
task='noise',
extension='.fif',
run=None)
# Save only the channel types we wish to analyze
# (same as for experimental data above).
raw_er_out.save(raw_er_fname_out, picks=chs_to_include,
overwrite=True)
del raw_er_out