def test_compute_whitener_rank(): """Test risky rank options.""" info = read_info(ave_fname) info = pick_info(info, pick_types(info, meg=True)) with info._unlock(): info['projs'] = [] # need a square version because the diag one takes shortcuts in # compute_whitener (users shouldn't even need this function so it's # private) cov = make_ad_hoc_cov(info)._as_square() assert len(cov['names']) == 306 _, _, rank = compute_whitener(cov, info, rank=None, return_rank=True) assert rank == 306 assert compute_rank(cov, info=info, verbose=True) == dict(meg=rank) cov['data'][-1] *= 1e-14 # trivially rank-deficient _, _, rank = compute_whitener(cov, info, rank=None, return_rank=True) assert rank == 305 assert compute_rank(cov, info=info, verbose=True) == dict(meg=rank) # this should emit a warning with pytest.warns(RuntimeWarning, match='exceeds the estimated'): _, _, rank = compute_whitener(cov, info, rank=dict(meg=306), return_rank=True) assert rank == 306
def _compute_rank(p, subj, run_indices): """Compute rank of the data.""" epochs_fnames, _ = get_epochs_evokeds_fnames(p, subj, p.analyses) _, fif_file = epochs_fnames epochs = read_epochs(fif_file) # .crop(p.bmin, p.bmax) maybe someday...? meg, eeg = 'meg' in epochs, 'eeg' in epochs rank = dict() epochs.apply_proj() if p.cov_rank_method == 'estimate_rank': # old way if meg: eps = epochs.copy().pick_types(meg=meg, eeg=False) eps = eps.get_data().transpose([1, 0, 2]) eps = eps.reshape(len(eps), -1) if 'grad' in epochs and 'mag' in epochs: # Neuromag key = 'meg' else: key = 'grad' if 'grad' in epochs else 'mag' rank[key] = estimate_rank(eps, tol=p.cov_rank_tol) if eeg: eps = epochs.copy().pick_types(meg=False, eeg=eeg) eps = eps.get_data().transpose([1, 0, 2]) eps = eps.reshape(len(eps), -1) rank['eeg'] = estimate_rank(eps, tol=p.cov_rank_tol) else: assert p.cov_rank_method == 'compute_rank' # new way rank = compute_rank(epochs, tol=p.cov_rank_tol, tol_kind='relative') for k, v in rank.items(): print(' : %s rank %2d' % (k.upper(), v), end='') return rank
def test_lcmv_maxfiltered(): """Test LCMV on maxfiltered data.""" raw = mne.io.read_raw_fif(fname_raw).fix_mag_coil_types() raw_sss = mne.preprocessing.maxwell_filter(raw) events = mne.find_events(raw_sss) del raw raw_sss.pick_types(meg='mag') assert len(raw_sss.ch_names) == 102 epochs = mne.Epochs(raw_sss, events) data_cov = mne.compute_covariance(epochs, tmin=0) fwd = mne.read_forward_solution(fname_fwd) rank = compute_rank(data_cov, info=epochs.info) assert rank == {'mag': 71} for use_rank in ('info', rank, 'full', None): make_lcmv(epochs.info, fwd, data_cov, rank=use_rank)
def run_maxwell_filter(subject, session=None): if config.proc and 'sss' in config.proc and config.use_maxwell_filter: raise ValueError(f'You cannot set use_maxwell_filter to True ' f'if data have already processed with Maxwell-filter.' f' Got proc={config.proc}.') bids_path_in = BIDSPath(subject=subject, session=session, task=config.get_task(), acquisition=config.acq, processing=config.proc, recording=config.rec, space=config.space, suffix=config.get_datatype(), datatype=config.get_datatype(), root=config.bids_root) bids_path_out = bids_path_in.copy().update(suffix='raw', root=config.deriv_root, check=False) # Load dev_head_t and digitization points from MaxFilter reference run. # Re-use in all runs and for processing empty-room recording. if config.use_maxwell_filter: reference_run = config.get_mf_reference_run() msg = f'Loading reference run: {reference_run}.' logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) bids_path_in.update(run=reference_run) info = mne.io.read_info(bids_path_in.fpath) dev_head_t = info['dev_head_t'] dig = info['dig'] del reference_run, info for run_idx, run in enumerate(config.get_runs()): bids_path_in.update(run=run) bids_path_out.update(run=run) raw = load_data(bids_path_in) # Fix stimulation artifact if config.fix_stim_artifact: events, _ = mne.events_from_annotations(raw) raw = mne.preprocessing.fix_stim_artifact( raw, events=events, event_id=None, tmin=config.stim_artifact_tmin, tmax=config.stim_artifact_tmax, mode='linear') # Auto-detect bad channels. if config.find_flat_channels_meg or config.find_noisy_channels_meg: find_bad_channels(raw=raw, subject=subject, session=session, task=config.get_task(), run=run) # Maxwell-filter experimental data. if config.use_maxwell_filter: msg = 'Applying Maxwell filter to experimental data.' logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) # Warn if no bad channels are set before Maxwell filter if not raw.info['bads']: msg = '\nFound no bad channels. \n ' logger.warning( gen_log_message(message=msg, subject=subject, step=1, session=session)) if config.mf_st_duration: msg = ' st_duration=%d' % (config.mf_st_duration) logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) # Keyword arguments shared between Maxwell filtering of the # experimental and the empty-room data. common_mf_kws = dict(calibration=get_mf_cal_fname( subject, session), cross_talk=get_mf_ctc_fname(subject, session), st_duration=config.mf_st_duration, origin=config.mf_head_origin, coord_frame='head', destination=dev_head_t) raw_sss = mne.preprocessing.maxwell_filter(raw, **common_mf_kws) raw_out = raw_sss raw_fname_out = (bids_path_out.copy().update(processing='sss', extension='.fif')) elif config.ch_types == ['eeg']: msg = 'Not applying Maxwell filter to EEG data.' logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) raw_out = raw raw_fname_out = bids_path_out.copy().update(extension='.fif') else: msg = ('Not applying Maxwell filter.\nIf you wish to apply it, ' 'set use_maxwell_filter=True in your configuration.') logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) raw_out = raw raw_fname_out = bids_path_out.copy().update(extension='.fif') # Save only the channel types we wish to analyze (including the # channels marked as "bad"). # We do not run `raw_out.pick()` here because it uses too much memory. chs_to_include = config.get_channels_to_analyze(raw_out.info) raw_out.save(raw_fname_out, picks=chs_to_include, overwrite=True, split_naming='bids') del raw_out if config.interactive: # Load the data we have just written, because it contains only # the relevant channels. raw = mne.io.read_raw_fif(raw_fname_out, allow_maxshield=True) raw.plot(n_channels=50, butterfly=True) # Empty-room processing. # # We pick the empty-room recording closest in time to the first run # of the experimental session. if run_idx == 0 and config.process_er: msg = 'Processing empty-room recording …' logger.info( gen_log_message(step=1, subject=subject, session=session, message=msg)) bids_path_er_in = bids_path_in.find_empty_room() raw_er = load_data(bids_path_er_in) raw_er.info['bads'] = [ ch for ch in raw.info['bads'] if ch.startswith('MEG') ] # Maxwell-filter empty-room data. if config.use_maxwell_filter: msg = 'Applying Maxwell filter to empty-room recording' logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) # We want to ensure we use the same coordinate frame origin in # empty-room and experimental data processing. To do this, we # inject the sensor locations and the head <> device transform # into the empty-room recording's info, and leave all other # parameters the same as for the experimental data. This is not # very clean, as we normally should not alter info manually, # except for info['bads']. Will need improvement upstream in # MNE-Python. raw_er.info['dig'] = dig raw_er.info['dev_head_t'] = dev_head_t raw_er_sss = mne.preprocessing.maxwell_filter( raw_er, **common_mf_kws) # Perform a sanity check: empty-room rank should match the # experimental data rank after Maxwell filtering. rank_exp = mne.compute_rank(raw, rank='info')['meg'] rank_er = mne.compute_rank(raw_er, rank='info')['meg'] if not np.isclose(rank_exp, rank_er): msg = (f'Experimental data rank {rank_exp:.1f} does not ' f'match empty-room data rank {rank_er:.1f} after ' f'Maxwell filtering. This indicates that the data ' f'were processed differenlty.') raise RuntimeError(msg) raw_er_out = raw_er_sss raw_er_fname_out = bids_path_out.copy().update( processing='sss') else: raw_er_out = raw_er raw_er_fname_out = bids_path_out.copy() raw_er_fname_out = raw_er_fname_out.update(task='noise', extension='.fif', run=None) # Save only the channel types we wish to analyze # (same as for experimental data above). raw_er_out.save(raw_er_fname_out, picks=chs_to_include, overwrite=True, split_naming='bids') del raw_er_out
def run_maxwell_filter(subject, session=None): deriv_path = config.get_subject_deriv_path(subject=subject, session=session, kind=config.get_kind()) os.makedirs(deriv_path, exist_ok=True) for run_idx, run in enumerate(config.get_runs()): bids_basename = make_bids_basename(subject=subject, session=session, task=config.get_task(), acquisition=config.acq, run=run, processing=config.proc, recording=config.rec, space=config.space) raw = load_data(bids_basename) if run_idx == 0: dev_head_t = raw.info['dev_head_t'] # Re-use in all runs. # Auto-detect bad channels. if config.find_flat_channels_meg or config.find_noisy_channels_meg: find_bad_channels(raw=raw, subject=subject, session=session, task=config.get_task(), run=run) # Maxwell-filter experimental data. if config.use_maxwell_filter: msg = 'Applying Maxwell filter to experimental data.' logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) # Warn if no bad channels are set before Maxwell filter if not raw.info['bads']: msg = '\nFound no bad channels. \n ' logger.warn( gen_log_message(message=msg, subject=subject, step=1, session=session)) if config.mf_st_duration: msg = ' st_duration=%d' % (config.mf_st_duration) logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) # Keyword arguments shared between Maxwell filtering of the # experimental and the empty-room data. common_mf_kws = dict(calibration=config.mf_cal_fname, cross_talk=config.mf_ctc_fname, st_duration=config.mf_st_duration, origin=config.mf_head_origin, coord_frame='head', destination=dev_head_t) raw_sss = mne.preprocessing.maxwell_filter(raw, **common_mf_kws) raw_out = raw_sss raw_fname_out = op.join(deriv_path, f'{bids_basename}_sss_raw.fif') else: msg = ('Not applying Maxwell filter.\nIf you wish to apply it, ' 'set use_maxwell_filter=True in your configuration.') logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) raw_out = raw raw_fname_out = op.join(deriv_path, f'{bids_basename}_nosss_raw.fif') raw_out.save(raw_fname_out, overwrite=True) if config.interactive: raw_out.plot(n_channels=50, butterfly=True) # Empty-room processing. # # We pick the empty-room recording closest in time to the first run # of the experimental session. if run_idx == 0 and config.noise_cov == 'emptyroom': msg = 'Processing empty-room recording …' logger.info( gen_log_message(step=1, subject=subject, session=session, message=msg)) bids_basename_er_in = get_matched_empty_room( bids_basename=bids_basename, bids_root=config.bids_root) raw_er = load_data(bids_basename_er_in) raw_er.info['bads'] = [ ch for ch in raw.info['bads'] if ch.startswith('MEG') ] # Maxwell-filter empty-room data. if config.use_maxwell_filter: msg = 'Applying Maxwell filter to empty-room recording' logger.info( gen_log_message(message=msg, step=1, subject=subject, session=session)) # We want to ensure we use the same coordinate frame origin in # empty-room and experimental data processing. To do this, we # inject the sensor locations and the head <> device transform # into the empty-room recording's info, and leave all other # parameters the same as for the experimental data. This is not # very clean, as we normally should not alter info manually, # except for info['bads']. Will need improvement upstream in # MNE-Python. raw_er.info['dig'] = raw.info['dig'] raw_er.info['dev_head_t'] = dev_head_t raw_er_sss = mne.preprocessing.maxwell_filter( raw_er, **common_mf_kws) # Perform a sanity check: empty-room rank should match the # experimental data rank after Maxwell filtering. rank_exp = mne.compute_rank(raw, rank='info')['meg'] rank_er = mne.compute_rank(raw_er, rank='info')['meg'] if not np.isclose(rank_exp, rank_er): msg = (f'Experimental data rank {rank_exp:.1f} does not ' f'match empty-room data rank {rank_er:.1f} after ' f'Maxwell filtering. This indicates that the data ' f'were processed differenlty.') raise RuntimeError(msg) raw_er_out = raw_er_sss raw_er_fname_out = op.join( deriv_path, f'{bids_basename}_emptyroom_sss_raw.fif') else: raw_er_out = raw_er raw_er_fname_out = op.join( deriv_path, f'{bids_basename}_emptyroom_nosss_raw.fif') raw_er_out.save(raw_er_fname_out, overwrite=True)
'sub-{}_task-{}-fwd.fif'.format(subject, task)) subjects_dir = op.join(data_path, 'derivatives', 'freesurfer', 'subjects') fwd = mne.read_forward_solution(fname_fwd) # %% # Compute covariances # ------------------- # ERS activity starts at 0.5 seconds after stimulus onset. Because these # data have been processed by MaxFilter directly (rather than MNE-Python's # version), we have to be careful to compute the rank with a more conservative # threshold in order to get the correct data rank (64). Once this is used in # combination with an advanced covariance estimator like "shrunk", the rank # will be correctly preserved. rank = mne.compute_rank(epochs, tol=1e-6, tol_kind='relative') active_win = (0.5, 1.5) baseline_win = (-1, 0) baseline_cov = compute_covariance(epochs, tmin=baseline_win[0], tmax=baseline_win[1], method='shrunk', rank=rank, verbose=True) active_cov = compute_covariance(epochs, tmin=active_win[0], tmax=active_win[1], method='shrunk', rank=rank, verbose=True)
def run_maxwell_filter(*, cfg, subject, session=None): if cfg.proc and 'sss' in cfg.proc and cfg.use_maxwell_filter: raise ValueError(f'You cannot set use_maxwell_filter to True ' f'if data have already processed with Maxwell-filter.' f' Got proc={config.proc}.') bids_path_out = BIDSPath(subject=subject, session=session, task=cfg.task, acquisition=cfg.acq, processing='sss', recording=cfg.rec, space=cfg.space, suffix='raw', extension='.fif', datatype=cfg.datatype, root=cfg.deriv_root, check=False) # Load dev_head_t and digitization points from MaxFilter reference run. # Re-use in all runs and for processing empty-room recording. msg = f'Loading reference run: {cfg.mf_reference_run}.' logger.info(**gen_log_kwargs(message=msg, subject=subject, session=session)) reference_run_info = get_reference_run_info( subject=subject, session=session, run=cfg.mf_reference_run ) dev_head_t = reference_run_info['dev_head_t'] dig = reference_run_info['dig'] del reference_run_info for run in cfg.runs: bids_path_out.update(run=run) raw = import_experimental_data( cfg=cfg, subject=subject, session=session, run=run, save=False ) # Maxwell-filter experimental data. msg = 'Applying Maxwell filter to experimental data.' logger.info(**gen_log_kwargs(message=msg, subject=subject, session=session, run=run)) # Warn if no bad channels are set before Maxwell filter # Create a copy, we'll need this later for setting the bads of the # empty-room recording bads = raw.info['bads'].copy() if not bads: msg = 'Found no bad channels.' logger.warning(**gen_log_kwargs(message=msg, subject=subject, session=session, run=run)) if cfg.mf_st_duration: msg = ' st_duration=%d' % (cfg.mf_st_duration) logger.info(**gen_log_kwargs(message=msg, subject=subject, session=session, run=run)) # Keyword arguments shared between Maxwell filtering of the # experimental and the empty-room data. common_mf_kws = dict( calibration=cfg.mf_cal_fname, cross_talk=cfg.mf_ctc_fname, st_duration=cfg.mf_st_duration, origin=cfg.mf_head_origin, coord_frame='head', destination=dev_head_t ) raw_sss = mne.preprocessing.maxwell_filter(raw, **common_mf_kws) # Save only the channel types we wish to analyze (including the # channels marked as "bad"). # We do not run `raw_sss.pick()` here because it uses too much memory. picks = config.get_channels_to_analyze(raw.info) raw_sss.save(bids_path_out, picks=picks, split_naming='bids', overwrite=True) del raw_sss if cfg.interactive: # Load the data we have just written, because it contains only # the relevant channels. raw = mne.io.read_raw_fif(bids_path_out, allow_maxshield=True) raw.plot(n_channels=50, butterfly=True) # Empty-room processing. # Only process empty-room data once – we ensure this by simply checking # if the current run is the reference run, and only then initiate # empty-room processing. No sophisticated logic behind this – it's just # convenient to code it this way. if cfg.process_er and run == cfg.mf_reference_run: msg = 'Processing empty-room recording …' logger.info(**gen_log_kwargs(subject=subject, session=session, message=msg)) raw_er = import_er_data( cfg=cfg, subject=subject, session=session, bads=bads, save=False ) # Maxwell-filter empty-room data. msg = 'Applying Maxwell filter to empty-room recording' logger.info(**gen_log_kwargs(message=msg, subject=subject, session=session)) # We want to ensure we use the same coordinate frame origin in # empty-room and experimental data processing. To do this, we # inject the sensor locations and the head <> device transform # into the empty-room recording's info, and leave all other # parameters the same as for the experimental data. This is not # very clean, as we normally should not alter info manually, # except for info['bads']. Will need improvement upstream in # MNE-Python. raw_er.info['dig'] = dig raw_er.info['dev_head_t'] = dev_head_t raw_er_sss = mne.preprocessing.maxwell_filter(raw_er, **common_mf_kws) # Perform a sanity check: empty-room rank should match the # experimental data rank after Maxwell filtering. raw_sss = mne.io.read_raw_fif(bids_path_out) rank_exp = mne.compute_rank(raw_sss, rank='info')['meg'] rank_er = mne.compute_rank(raw_er_sss, rank='info')['meg'] if not np.isclose(rank_exp, rank_er): msg = (f'Experimental data rank {rank_exp:.1f} does not ' f'match empty-room data rank {rank_er:.1f} after ' f'Maxwell filtering. This indicates that the data ' f'were processed differently.') raise RuntimeError(msg) raw_er_fname_out = bids_path_out.copy().update( task='noise', run=None, processing='sss' ) # Save only the channel types we wish to analyze # (same as for experimental data above). raw_er_sss.save(raw_er_fname_out, picks=picks, overwrite=True, split_naming='bids') del raw_er_sss
import os.path as op import numpy as np import matplotlib.pyplot as plt import mne import config_drago as cfg info = np.load(op.join(cfg.path_data, 'info_allch.npy')).item() picks = mne.pick_types(info, meg='mag') fname = op.join(cfg.path_outputs, 'covs_allch_oas.h5') data = mne.externals.h5io.read_hdf5(fname) # (sub, fb, ch, ch) subjects = [d['subject'] for d in data if 'subject' in d] covs = [d['covs'][:, picks][:, :, picks] for d in data if 'subject' in d] covs = np.array(covs) # (sub,fb,chan,chan) ranks = [] for sub in range(len(subjects)): cov = mne.Covariance(covs[sub][4], np.array(info['ch_names'])[picks], [], [], 1) ranks.append(mne.compute_rank(cov, info=info)['mag']) plt.figure() plt.hist(ranks)
def run_ica_correction(sub_id, raw, method='picard', reject=None, decim=3, random_state=42, show_figs=False, results_dir=None): """ Fit and apply ICA to correct heartbeats and blinks Parameters ---------- sub_id: str Subject ID raw: mne.Raw object Data to apply ICA to method: str, default 'picard' Method for the ICA algorithm. See the documentation on the MNE webpage for other options reject: None or dict, default None Reject values previous to the ICA fit and application decim: int, default 3 ICA parameter. Check out the documentation for more info random_state: int, default 42 Seed for ICA random state Returns ------- sub_id: str Subject ID, unchanged raw: mne.Raw object Our old friend, but now without blinks and hearbeats (hopefully) ica: mne.ICA object The ICA object with the configuration used to eliminate artifacts """ # picks_meg = mne.pick_types(raw.info, meg=True, eog=True, stim=True, # exclude='bads') # The rank will be used as a reference for the ICA components rank = mne.compute_rank(raw) ica = ICA(n_components=rank['meg'], method=method, random_state=random_state) ica.fit(raw, decim=decim, reject=reject) # {'mag': 5e-12, 'grad': 4000e-13} # Create eog/ecg epochs using our dedicated channels in the data, and then find data # segments containing artifacts eog_epochs = create_eog_epochs(raw, ch_name='EOG061', reject=None) eog_inds, _ = ica.find_bads_eog(eog_epochs) ecg_epochs = create_ecg_epochs(raw, reject=None) ecg_inds, _ = ica.find_bads_ecg(ecg_epochs) # Compute an average of our eog/ecg epochs eog_avg = eog_epochs.average() ecg_avg = ecg_epochs.average() if show_figs or results_dir is not None: fig_eog = ica.plot_overlay(eog_avg, exclude=eog_inds, show=show_figs) fig_ecg = ica.plot_overlay(ecg_avg, exclude=ecg_inds, show=show_figs) if results_dir is not None: sub_dir = os.path.join(results_dir, sub_id) # Create a dir with the ID name if not os.path.exists(sub_dir): os.mkdir(sub_dir) fig_eog.savefig(os.path.join(sub_dir, f'{sub_id}_eog_correction.png')) fig_ecg.savefig(os.path.join(sub_dir, f'{sub_id}_ecg_correction.png')) # Tell our friend ICA what to do, and apply to data ica.exclude.extend(eog_inds) ica.exclude.extend(ecg_inds) ica.apply(raw) return sub_id, raw, ica