def test_windows_from_events_(lazy_loadable_dataset): msg = '"trial_stop_offset_samples" too large\\. Stop of last trial ' \ '\\(19900\\) \\+ "trial_stop_offset_samples" \\(250\\) must be ' \ 'smaller than length of recording \\(20000\\)\\.' with pytest.raises(ValueError, match=msg): create_windows_from_events( concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0, trial_stop_offset_samples=250, window_size_samples=100, window_stride_samples=100, drop_last_window=False)
def test_overlapping_trial_offsets(concat_ds_targets): concat_ds, _ = concat_ds_targets with pytest.raises(NotImplementedError, match='Trial overlap not implemented.'): create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=-2000, trial_stop_offset_samples=0, window_size_samples=1000, window_stride_samples=1000, drop_last_window=False)
def test_windows_from_events_different_events(tmpdir_factory): description_expected = 5 * ['T0', 'T1'] + 4 * ['T2', 'T3'] + 2 * ['T1'] raw = _get_raw(tmpdir_factory, description_expected[:10]) base_ds = BaseDataset(raw, description=pd.Series({'file_id': 1})) raw_1 = _get_raw(tmpdir_factory, description_expected[10:]) base_ds_1 = BaseDataset(raw_1, description=pd.Series({'file_id': 2})) concat_ds = BaseConcatDataset([base_ds, base_ds_1]) windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False) description = [] events = [] for ds in windows.datasets: description += ds.windows.metadata['target'].to_list() events += ds.windows.events[:, 0].tolist() assert len(description) == 20 np.testing.assert_array_equal(description, 5 * [0, 1] + 4 * [2, 3] + 2 * [1]) np.testing.assert_array_equal( np.concatenate( [raw.time_as_index(raw.annotations.onset, use_rounding=True), raw_1.time_as_index(raw.annotations.onset, use_rounding=True)]), events)
def test_epochs_kwargs(lazy_loadable_dataset): picks = ['ch0'] on_missing = 'warning' flat = {'eeg': 3e-6} reject = {'eeg': 43e-6} windows = create_windows_from_events( concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False, picks=picks, on_missing=on_missing, flat=flat, reject=reject) epochs = windows.datasets[0].windows assert epochs.ch_names == picks assert epochs.reject == reject assert epochs.flat == flat windows = create_fixed_length_windows( concat_ds=lazy_loadable_dataset, start_offset_samples=0, stop_offset_samples=None, window_size_samples=100, window_stride_samples=100, drop_last_window=False, picks=picks, on_missing=on_missing, flat=flat, reject=reject) epochs = windows.datasets[0].windows assert epochs.ch_names == picks assert epochs.reject == reject assert epochs.flat == flat
def test_drop_bad_windows(concat_ds_targets, drop_bad_windows, preload): concat_ds, _ = concat_ds_targets windows_from_events = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False, preload=preload, drop_bad_windows=drop_bad_windows) windows_fixed_length = create_fixed_length_windows( concat_ds=concat_ds, start_offset_samples=0, stop_offset_samples=1000, window_size_samples=1000, window_stride_samples=1000, drop_last_window=False, preload=preload, drop_bad_windows=drop_bad_windows) assert (windows_from_events.datasets[0].windows._bad_dropped == drop_bad_windows) assert (windows_fixed_length.datasets[0].windows._bad_dropped == drop_bad_windows)
def test_windows_from_events_preload_false(lazy_loadable_dataset): windows = create_windows_from_events( concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False) assert all([not ds.windows.preload for ds in windows.datasets])
def test_one_supercrop_per_original_trial(concat_ds_targets): concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, supercrop_size_samples=1000, supercrop_stride_samples=1, drop_samples=False) description = windows.datasets[0].windows.metadata["target"].to_list() assert len(description) == len(targets) np.testing.assert_array_equal(description, targets)
def test_stride_has_no_effect(concat_ds_targets): concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=1000, window_stride_samples=1000, drop_last_window=False) description = windows.datasets[0].windows.metadata["target"].to_list() assert len(description) == len(targets) np.testing.assert_array_equal(description, targets)
def test_dropping_last_incomplete_window(concat_ds_targets): concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=-250, trial_stop_offset_samples=-750, window_size_samples=250, window_stride_samples=300, drop_last_window=True) description = windows.datasets[0].windows.metadata["target"].to_list() assert len(description) == len(targets) np.testing.assert_array_equal(description, targets)
def test_windows_from_events_(lazy_loadable_dataset): with pytest.raises(ValueError, match='"trial_stop_offset_samples" too large\\. Stop ' 'of last trial \\(19900\\) \\+ ' '"trial_stop_offset_samples" \\(250\\) must be ' 'smaller then length of recording 20000\\.' ): windows = create_windows_from_events( concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0, trial_stop_offset_samples=250, supercrop_size_samples=100, supercrop_stride_samples=100, drop_samples=False)
def test_shifting_last_supercrop_back_in(concat_ds_targets): concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=-250, trial_stop_offset_samples=-750, supercrop_size_samples=250, supercrop_stride_samples=300, drop_samples=False) description = windows.datasets[0].windows.metadata["target"].to_list() assert len(description) == len(targets) * 2 np.testing.assert_array_equal(description[0::2], targets) np.testing.assert_array_equal(description[1::2], targets)
def test_maximally_overlapping_windows(concat_ds_targets): concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=-2, trial_stop_offset_samples=0, window_size_samples=1000, window_stride_samples=1, drop_last_window=False) description = windows.datasets[0].windows.metadata["target"].to_list() assert len(description) == len(targets) * 3 np.testing.assert_array_equal(description[0::3], targets) np.testing.assert_array_equal(description[1::3], targets) np.testing.assert_array_equal(description[2::3], targets)
def test_windows_from_events_mapping_filter(tmpdir_factory): raw = _get_raw(tmpdir_factory, 5 * ['T0', 'T1']) base_ds = BaseDataset(raw, description=pd.Series({'file_id': 1})) concat_ds = BaseConcatDataset([base_ds]) windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False, mapping={'T1': 0}) description = windows.datasets[0].windows.metadata['target'].to_list() assert len(description) == 5 np.testing.assert_array_equal(description, np.zeros(5)) # dataset should contain only 'T1' events np.testing.assert_array_equal( (raw.time_as_index(raw.annotations.onset[1::2], use_rounding=True)), windows.datasets[0].windows.events[:, 0])
def test_windows_from_events_n_jobs(lazy_loadable_dataset): longer_dataset = BaseConcatDataset([lazy_loadable_dataset.datasets[0]] * 8) windows = [create_windows_from_events( concat_ds=longer_dataset, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False, preload=True, n_jobs=n_jobs) for n_jobs in [1, 2]] assert windows[0].description.equals(windows[1].description) for ds1, ds2 in zip(windows[0].datasets, windows[1].datasets): # assert ds1.windows == ds2.windows # Runs locally, fails in CI assert np.allclose(ds1.windows.get_data(), ds2.windows.get_data()) assert pd.Series(ds1.windows.info).to_json() == \ pd.Series(ds2.windows.info).to_json() assert ds1.description.equals(ds2.description) assert np.array_equal(ds1.y, ds2.y) assert np.array_equal(ds1.crop_inds, ds2.crop_inds)
def build_epoch(subjects, recording, crop_wake_mins, preprocessing, train=True): dataset = SleepPhysionet(subject_ids=subjects, recording_ids=recording, crop_wake_mins=crop_wake_mins) if preprocessing: preprocessors = [] if "microvolt_scaling" in preprocessing: preprocessors.append(NumpyPreproc(fn=lambda x: x * 1e6)) if "filtering" in preprocessing: high_cut_hz = 30 preprocessors.append( MNEPreproc(fn='filter', l_freq=None, h_freq=high_cut_hz)) # Transform the data preprocess(dataset, preprocessors) mapping = { # We merge stages 3 and 4 following AASM standards. 'Sleep stage W': 0, 'Sleep stage 1': 1, 'Sleep stage 2': 2, 'Sleep stage 3': 3, 'Sleep stage 4': 3, 'Sleep stage R': 4 } window_size_s = 30 sfreq = 100 window_size_samples = window_size_s * sfreq windows_dataset = create_windows_from_events( dataset, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=window_size_samples, window_stride_samples=window_size_samples, preload=True, mapping=mapping) return windows_dataset
def test_single_sample_size_windows(concat_ds_targets): concat_ds, targets = concat_ds_targets # reduce dataset for faster test, only first 3 events targets = targets[:3] underlying_raw = concat_ds.datasets[0].raw annotations = underlying_raw.annotations underlying_raw.set_annotations(annotations[:3]) # have to supply explicit mapping as only two classes appear in first 3 # targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=1, window_stride_samples=1, drop_last_window=False, mapping=dict(tongue=3, left_hand=1, right_hand=2, feet=4)) description = windows.datasets[0].windows.metadata["target"].to_list() assert len(description) == len(targets) * 1000 np.testing.assert_array_equal(description[::1000], targets) np.testing.assert_array_equal(description[999::1000], targets)
def test_windows_from_events_cropped(lazy_loadable_dataset): """Test windowing from events on cropped data. Cropping raw data changes the `first_samp` attribute of the Raw object, and so it is important to test this is taken into account by the windowers. """ tmin, tmax = 100, 120 ds = copy.deepcopy(lazy_loadable_dataset) ds.datasets[0].raw.annotations.crop(tmin, tmax) crop_ds = copy.deepcopy(lazy_loadable_dataset) crop_transform = MNEPreproc('crop', tmin=tmin, tmax=tmax) preprocess(crop_ds, [crop_transform]) # Extract windows windows1 = create_windows_from_events(concat_ds=ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False) windows2 = create_windows_from_events(concat_ds=crop_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, window_size_samples=100, window_stride_samples=100, drop_last_window=False) assert (windows1[0][0] == windows2[0][0]).all() # Make sure events that fall outside of recording will trigger an error with pytest.raises(ValueError, match='"trial_stop_offset_samples" too large'): create_windows_from_events(concat_ds=ds, trial_start_offset_samples=0, trial_stop_offset_samples=10000, window_size_samples=100, window_stride_samples=100, drop_last_window=False) with pytest.raises(ValueError, match='"trial_stop_offset_samples" too large'): create_windows_from_events(concat_ds=crop_ds, trial_start_offset_samples=0, trial_stop_offset_samples=2001, window_size_samples=100, window_stride_samples=100, drop_last_window=False)
("pick_types", dict(eeg=True, meg=False, stim=False)), ('apply_function', dict(fun=lambda x: x * 1e6, channel_wise=False)), ('filter', dict(l_freq=low_cut_hz, h_freq=high_cut_hz)), ('apply_function', dict(fun=standardize_func, channel_wise=False)) ] transform_concat_ds(dataset, raw_transform_dict) sfreqs = [ds.raw.info['sfreq'] for ds in dataset.datasets] assert len(np.unique(sfreqs)) == 1 trial_start_offset_samples = int(trial_start_offset_seconds * sfreqs[0]) windows_dataset = create_windows_from_events( dataset, trial_start_offset_samples=trial_start_offset_samples, trial_stop_offset_samples=0, supercrop_size_samples=input_time_length, supercrop_stride_samples=input_time_length, drop_samples=False, preload=True, ) splitted = windows_dataset.split('session') train_set = splitted['session_T'] valid_set = splitted['session_E'] clf = EEGClassifier( model, cropped=False, criterion=torch.nn.NLLLoss, optimizer=torch.optim.AdamW, train_split=predefined_split(valid_set),
def test_window_sizes_from_events(concat_ds_targets): # no fixed window size, no offsets expected_n_samples = 1000 concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=0, drop_last_window=False) x, y, ind = windows[0] assert x.shape[-1] == ind[-1] - ind[-2] assert x.shape[-1] == expected_n_samples # no fixed window size, positive trial start offset expected_n_samples = 999 concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=1, trial_stop_offset_samples=0, drop_last_window=False) x, y, ind = windows[0] assert x.shape[-1] == ind[-1] - ind[-2] assert x.shape[-1] == expected_n_samples # no fixed window size, negative trial start offset expected_n_samples = 1001 concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=-1, trial_stop_offset_samples=0, drop_last_window=False) x, y, ind = windows[0] assert x.shape[-1] == ind[-1] - ind[-2] assert x.shape[-1] == expected_n_samples # no fixed window size, positive trial stop offset expected_n_samples = 1001 concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=1, drop_last_window=False) x, y, ind = windows[0] assert x.shape[-1] == ind[-1] - ind[-2] assert x.shape[-1] == expected_n_samples # no fixed window size, negative trial stop offset expected_n_samples = 999 concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=0, trial_stop_offset_samples=-1, drop_last_window=False) x, y, ind = windows[0] assert x.shape[-1] == ind[-1] - ind[-2] assert x.shape[-1] == expected_n_samples # fixed window size, trial offsets should not change window size expected_n_samples = 250 concat_ds, targets = concat_ds_targets windows = create_windows_from_events( concat_ds=concat_ds, trial_start_offset_samples=3, trial_stop_offset_samples=8, window_size_samples=250, window_stride_samples=250, drop_last_window=False) x, y, ind = windows[0] assert x.shape[-1] == ind[-1] - ind[-2] assert x.shape[-1] == expected_n_samples