def test_windows_from_events_(lazy_loadable_dataset):
msg = '"trial_stop_offset_samples" too large\\. Stop of last trial ' \
'\\(19900\\) \\+ "trial_stop_offset_samples" \\(250\\) must be ' \
'smaller than length of recording \\(20000\\)\\.'
with pytest.raises(ValueError, match=msg):
create_windows_from_events(
concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0,
trial_stop_offset_samples=250, window_size_samples=100,
window_stride_samples=100, drop_last_window=False)
def test_overlapping_trial_offsets(concat_ds_targets):
concat_ds, _ = concat_ds_targets
with pytest.raises(NotImplementedError,
match='Trial overlap not implemented.'):
create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=-2000, trial_stop_offset_samples=0,
window_size_samples=1000, window_stride_samples=1000,
drop_last_window=False)
def test_windows_from_events_different_events(tmpdir_factory):
description_expected = 5 * ['T0', 'T1'] + 4 * ['T2', 'T3'] + 2 * ['T1']
raw = _get_raw(tmpdir_factory, description_expected[:10])
base_ds = BaseDataset(raw, description=pd.Series({'file_id': 1}))
raw_1 = _get_raw(tmpdir_factory, description_expected[10:])
base_ds_1 = BaseDataset(raw_1, description=pd.Series({'file_id': 2}))
concat_ds = BaseConcatDataset([base_ds, base_ds_1])
windows = create_windows_from_events(
concat_ds=concat_ds, trial_start_offset_samples=0,
trial_stop_offset_samples=0, window_size_samples=100,
window_stride_samples=100, drop_last_window=False)
description = []
events = []
for ds in windows.datasets:
description += ds.windows.metadata['target'].to_list()
events += ds.windows.events[:, 0].tolist()
assert len(description) == 20
np.testing.assert_array_equal(description,
5 * [0, 1] + 4 * [2, 3] + 2 * [1])
np.testing.assert_array_equal(
np.concatenate(
[raw.time_as_index(raw.annotations.onset, use_rounding=True),
raw_1.time_as_index(raw.annotations.onset, use_rounding=True)]),
events)
def test_epochs_kwargs(lazy_loadable_dataset):
picks = ['ch0']
on_missing = 'warning'
flat = {'eeg': 3e-6}
reject = {'eeg': 43e-6}
windows = create_windows_from_events(
concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0,
trial_stop_offset_samples=0, window_size_samples=100,
window_stride_samples=100, drop_last_window=False, picks=picks,
on_missing=on_missing, flat=flat, reject=reject)
epochs = windows.datasets[0].windows
assert epochs.ch_names == picks
assert epochs.reject == reject
assert epochs.flat == flat
windows = create_fixed_length_windows(
concat_ds=lazy_loadable_dataset, start_offset_samples=0,
stop_offset_samples=None, window_size_samples=100,
window_stride_samples=100, drop_last_window=False, picks=picks,
on_missing=on_missing, flat=flat, reject=reject)
epochs = windows.datasets[0].windows
assert epochs.ch_names == picks
assert epochs.reject == reject
assert epochs.flat == flat
def test_drop_bad_windows(concat_ds_targets, drop_bad_windows, preload):
concat_ds, _ = concat_ds_targets
windows_from_events = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0,
trial_stop_offset_samples=0,
window_size_samples=100,
window_stride_samples=100,
drop_last_window=False,
preload=preload,
drop_bad_windows=drop_bad_windows)
windows_fixed_length = create_fixed_length_windows(
concat_ds=concat_ds,
start_offset_samples=0,
stop_offset_samples=1000,
window_size_samples=1000,
window_stride_samples=1000,
drop_last_window=False,
preload=preload,
drop_bad_windows=drop_bad_windows)
assert (windows_from_events.datasets[0].windows._bad_dropped ==
drop_bad_windows)
assert (windows_fixed_length.datasets[0].windows._bad_dropped ==
drop_bad_windows)
def test_windows_from_events_preload_false(lazy_loadable_dataset):
windows = create_windows_from_events(
concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0,
trial_stop_offset_samples=0, window_size_samples=100,
window_stride_samples=100, drop_last_window=False)
assert all([not ds.windows.preload for ds in windows.datasets])
def test_one_supercrop_per_original_trial(concat_ds_targets):
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0, trial_stop_offset_samples=0,
supercrop_size_samples=1000, supercrop_stride_samples=1,
drop_samples=False)
description = windows.datasets[0].windows.metadata["target"].to_list()
assert len(description) == len(targets)
np.testing.assert_array_equal(description, targets)
def test_stride_has_no_effect(concat_ds_targets):
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0, trial_stop_offset_samples=0,
window_size_samples=1000, window_stride_samples=1000,
drop_last_window=False)
description = windows.datasets[0].windows.metadata["target"].to_list()
assert len(description) == len(targets)
np.testing.assert_array_equal(description, targets)
def test_dropping_last_incomplete_window(concat_ds_targets):
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=-250, trial_stop_offset_samples=-750,
window_size_samples=250, window_stride_samples=300,
drop_last_window=True)
description = windows.datasets[0].windows.metadata["target"].to_list()
assert len(description) == len(targets)
np.testing.assert_array_equal(description, targets)
def test_windows_from_events_(lazy_loadable_dataset):
with pytest.raises(ValueError,
match='"trial_stop_offset_samples" too large\\. Stop '
'of last trial \\(19900\\) \\+ '
'"trial_stop_offset_samples" \\(250\\) must be '
'smaller then length of recording 20000\\.'
):
windows = create_windows_from_events(
concat_ds=lazy_loadable_dataset, trial_start_offset_samples=0,
trial_stop_offset_samples=250, supercrop_size_samples=100,
supercrop_stride_samples=100, drop_samples=False)
def test_shifting_last_supercrop_back_in(concat_ds_targets):
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=-250, trial_stop_offset_samples=-750,
supercrop_size_samples=250, supercrop_stride_samples=300,
drop_samples=False)
description = windows.datasets[0].windows.metadata["target"].to_list()
assert len(description) == len(targets) * 2
np.testing.assert_array_equal(description[0::2], targets)
np.testing.assert_array_equal(description[1::2], targets)
def test_maximally_overlapping_windows(concat_ds_targets):
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=-2, trial_stop_offset_samples=0,
window_size_samples=1000, window_stride_samples=1,
drop_last_window=False)
description = windows.datasets[0].windows.metadata["target"].to_list()
assert len(description) == len(targets) * 3
np.testing.assert_array_equal(description[0::3], targets)
np.testing.assert_array_equal(description[1::3], targets)
np.testing.assert_array_equal(description[2::3], targets)
def test_windows_from_events_mapping_filter(tmpdir_factory):
raw = _get_raw(tmpdir_factory, 5 * ['T0', 'T1'])
base_ds = BaseDataset(raw, description=pd.Series({'file_id': 1}))
concat_ds = BaseConcatDataset([base_ds])
windows = create_windows_from_events(
concat_ds=concat_ds, trial_start_offset_samples=0,
trial_stop_offset_samples=0, window_size_samples=100,
window_stride_samples=100, drop_last_window=False, mapping={'T1': 0})
description = windows.datasets[0].windows.metadata['target'].to_list()
assert len(description) == 5
np.testing.assert_array_equal(description, np.zeros(5))
# dataset should contain only 'T1' events
np.testing.assert_array_equal(
(raw.time_as_index(raw.annotations.onset[1::2], use_rounding=True)),
windows.datasets[0].windows.events[:, 0])
def test_windows_from_events_n_jobs(lazy_loadable_dataset):
longer_dataset = BaseConcatDataset([lazy_loadable_dataset.datasets[0]] * 8)
windows = [create_windows_from_events(
concat_ds=longer_dataset, trial_start_offset_samples=0,
trial_stop_offset_samples=0, window_size_samples=100,
window_stride_samples=100, drop_last_window=False, preload=True,
n_jobs=n_jobs) for n_jobs in [1, 2]]
assert windows[0].description.equals(windows[1].description)
for ds1, ds2 in zip(windows[0].datasets, windows[1].datasets):
# assert ds1.windows == ds2.windows # Runs locally, fails in CI
assert np.allclose(ds1.windows.get_data(), ds2.windows.get_data())
assert pd.Series(ds1.windows.info).to_json() == \
pd.Series(ds2.windows.info).to_json()
assert ds1.description.equals(ds2.description)
assert np.array_equal(ds1.y, ds2.y)
assert np.array_equal(ds1.crop_inds, ds2.crop_inds)
def build_epoch(subjects,
recording,
crop_wake_mins,
preprocessing,
train=True):
dataset = SleepPhysionet(subject_ids=subjects,
recording_ids=recording,
crop_wake_mins=crop_wake_mins)
if preprocessing:
preprocessors = []
if "microvolt_scaling" in preprocessing:
preprocessors.append(NumpyPreproc(fn=lambda x: x * 1e6))
if "filtering" in preprocessing:
high_cut_hz = 30
preprocessors.append(
MNEPreproc(fn='filter', l_freq=None, h_freq=high_cut_hz))
# Transform the data
preprocess(dataset, preprocessors)
mapping = { # We merge stages 3 and 4 following AASM standards.
'Sleep stage W': 0,
'Sleep stage 1': 1,
'Sleep stage 2': 2,
'Sleep stage 3': 3,
'Sleep stage 4': 3,
'Sleep stage R': 4
}
window_size_s = 30
sfreq = 100
window_size_samples = window_size_s * sfreq
windows_dataset = create_windows_from_events(
dataset,
trial_start_offset_samples=0,
trial_stop_offset_samples=0,
window_size_samples=window_size_samples,
window_stride_samples=window_size_samples,
preload=True,
mapping=mapping)
return windows_dataset
def test_single_sample_size_windows(concat_ds_targets):
concat_ds, targets = concat_ds_targets
# reduce dataset for faster test, only first 3 events
targets = targets[:3]
underlying_raw = concat_ds.datasets[0].raw
annotations = underlying_raw.annotations
underlying_raw.set_annotations(annotations[:3])
# have to supply explicit mapping as only two classes appear in first 3
# targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0, trial_stop_offset_samples=0,
window_size_samples=1, window_stride_samples=1,
drop_last_window=False, mapping=dict(tongue=3, left_hand=1,
right_hand=2, feet=4))
description = windows.datasets[0].windows.metadata["target"].to_list()
assert len(description) == len(targets) * 1000
np.testing.assert_array_equal(description[::1000], targets)
np.testing.assert_array_equal(description[999::1000], targets)
def test_windows_from_events_cropped(lazy_loadable_dataset):
"""Test windowing from events on cropped data.
Cropping raw data changes the `first_samp` attribute of the Raw object, and
so it is important to test this is taken into account by the windowers.
"""
tmin, tmax = 100, 120
ds = copy.deepcopy(lazy_loadable_dataset)
ds.datasets[0].raw.annotations.crop(tmin, tmax)
crop_ds = copy.deepcopy(lazy_loadable_dataset)
crop_transform = MNEPreproc('crop', tmin=tmin, tmax=tmax)
preprocess(crop_ds, [crop_transform])
# Extract windows
windows1 = create_windows_from_events(concat_ds=ds,
trial_start_offset_samples=0,
trial_stop_offset_samples=0,
window_size_samples=100,
window_stride_samples=100,
drop_last_window=False)
windows2 = create_windows_from_events(concat_ds=crop_ds,
trial_start_offset_samples=0,
trial_stop_offset_samples=0,
window_size_samples=100,
window_stride_samples=100,
drop_last_window=False)
assert (windows1[0][0] == windows2[0][0]).all()
# Make sure events that fall outside of recording will trigger an error
with pytest.raises(ValueError,
match='"trial_stop_offset_samples" too large'):
create_windows_from_events(concat_ds=ds,
trial_start_offset_samples=0,
trial_stop_offset_samples=10000,
window_size_samples=100,
window_stride_samples=100,
drop_last_window=False)
with pytest.raises(ValueError,
match='"trial_stop_offset_samples" too large'):
create_windows_from_events(concat_ds=crop_ds,
trial_start_offset_samples=0,
trial_stop_offset_samples=2001,
window_size_samples=100,
window_stride_samples=100,
drop_last_window=False)
("pick_types", dict(eeg=True, meg=False, stim=False)),
('apply_function', dict(fun=lambda x: x * 1e6, channel_wise=False)),
('filter', dict(l_freq=low_cut_hz, h_freq=high_cut_hz)),
('apply_function', dict(fun=standardize_func, channel_wise=False))
]
transform_concat_ds(dataset, raw_transform_dict)
sfreqs = [ds.raw.info['sfreq'] for ds in dataset.datasets]
assert len(np.unique(sfreqs)) == 1
trial_start_offset_samples = int(trial_start_offset_seconds * sfreqs[0])
windows_dataset = create_windows_from_events(
dataset,
trial_start_offset_samples=trial_start_offset_samples,
trial_stop_offset_samples=0,
supercrop_size_samples=input_time_length,
supercrop_stride_samples=input_time_length,
drop_samples=False,
preload=True,
)
splitted = windows_dataset.split('session')
train_set = splitted['session_T']
valid_set = splitted['session_E']
clf = EEGClassifier(
model,
cropped=False,
criterion=torch.nn.NLLLoss,
optimizer=torch.optim.AdamW,
train_split=predefined_split(valid_set),
def test_window_sizes_from_events(concat_ds_targets):
# no fixed window size, no offsets
expected_n_samples = 1000
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0, trial_stop_offset_samples=0,
drop_last_window=False)
x, y, ind = windows[0]
assert x.shape[-1] == ind[-1] - ind[-2]
assert x.shape[-1] == expected_n_samples
# no fixed window size, positive trial start offset
expected_n_samples = 999
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=1, trial_stop_offset_samples=0,
drop_last_window=False)
x, y, ind = windows[0]
assert x.shape[-1] == ind[-1] - ind[-2]
assert x.shape[-1] == expected_n_samples
# no fixed window size, negative trial start offset
expected_n_samples = 1001
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=-1, trial_stop_offset_samples=0,
drop_last_window=False)
x, y, ind = windows[0]
assert x.shape[-1] == ind[-1] - ind[-2]
assert x.shape[-1] == expected_n_samples
# no fixed window size, positive trial stop offset
expected_n_samples = 1001
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0, trial_stop_offset_samples=1,
drop_last_window=False)
x, y, ind = windows[0]
assert x.shape[-1] == ind[-1] - ind[-2]
assert x.shape[-1] == expected_n_samples
# no fixed window size, negative trial stop offset
expected_n_samples = 999
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=0, trial_stop_offset_samples=-1,
drop_last_window=False)
x, y, ind = windows[0]
assert x.shape[-1] == ind[-1] - ind[-2]
assert x.shape[-1] == expected_n_samples
# fixed window size, trial offsets should not change window size
expected_n_samples = 250
concat_ds, targets = concat_ds_targets
windows = create_windows_from_events(
concat_ds=concat_ds,
trial_start_offset_samples=3, trial_stop_offset_samples=8,
window_size_samples=250, window_stride_samples=250,
drop_last_window=False)
x, y, ind = windows[0]
assert x.shape[-1] == ind[-1] - ind[-2]
assert x.shape[-1] == expected_n_samples
