def test_save_concat_windows_dataset(setup_concat_windows_dataset, tmpdir):
concat_windows_dataset = setup_concat_windows_dataset
n_windows_datasets = len(concat_windows_dataset.datasets)
save_concat_dataset(path=tmpdir,
concat_dataset=concat_windows_dataset,
overwrite=False)
assert os.path.exists(tmpdir.join(f"description.json"))
for windows_i in range(n_windows_datasets):
assert os.path.exists(tmpdir.join(f"{windows_i}-epo.fif"))
assert not os.path.exists(tmpdir.join(f"{n_windows_datasets}-epo.fif"))
def test_load_multiple_concat_raw_dataset(setup_concat_raw_dataset, tmpdir):
concat_raw_dataset = setup_concat_raw_dataset
for i in range(2):
path = os.path.join(tmpdir, str(i))
os.makedirs(path)
save_concat_dataset(path=path,
concat_dataset=concat_raw_dataset,
overwrite=False)
loaded_concat_raw_datasets = load_concat_dataset(path=tmpdir,
preload=False)
assert 2 * len(concat_raw_dataset) == len(loaded_concat_raw_datasets)
assert (2 * len(concat_raw_dataset.datasets) == len(
loaded_concat_raw_datasets.datasets))
assert (2 * len(concat_raw_dataset.description) == len(
loaded_concat_raw_datasets.description))
def test_load_concat_raw_dataset(setup_concat_raw_dataset, tmpdir):
concat_raw_dataset = setup_concat_raw_dataset
n_raw_datasets = len(concat_raw_dataset.datasets)
save_concat_dataset(path=tmpdir,
concat_dataset=concat_raw_dataset,
overwrite=False)
loaded_concat_raw_dataset = load_concat_dataset(path=tmpdir, preload=False)
assert len(concat_raw_dataset) == len(loaded_concat_raw_dataset)
assert (len(concat_raw_dataset.datasets) == len(
loaded_concat_raw_dataset.datasets))
assert (len(concat_raw_dataset.description) == len(
loaded_concat_raw_dataset.description))
for raw_i in range(n_raw_datasets):
actual_x, actual_y = concat_raw_dataset[raw_i]
x, y = loaded_concat_raw_dataset[raw_i]
np.testing.assert_allclose(x, actual_x, rtol=1e-4, atol=1e-5)
pd.testing.assert_frame_equal(concat_raw_dataset.description,
loaded_concat_raw_dataset.description)
def test_load_concat_windows_dataset(setup_concat_windows_dataset, tmpdir):
concat_windows_dataset = setup_concat_windows_dataset
n_windows_datasets = len(concat_windows_dataset.datasets)
save_concat_dataset(path=tmpdir,
concat_dataset=concat_windows_dataset,
overwrite=False)
loaded_concat_windows_dataset = load_concat_dataset(path=tmpdir,
preload=False)
assert len(concat_windows_dataset) == len(loaded_concat_windows_dataset)
assert (len(concat_windows_dataset.datasets) == len(
loaded_concat_windows_dataset.datasets))
assert (len(concat_windows_dataset.description) == len(
loaded_concat_windows_dataset.description))
for windows_i in range(n_windows_datasets):
actual_x, actual_y, actual_crop_inds = concat_windows_dataset[
windows_i]
x, y, crop_inds = loaded_concat_windows_dataset[windows_i]
np.testing.assert_allclose(x, actual_x, rtol=1e-4, atol=1e-5)
np.testing.assert_allclose(y, actual_y, rtol=1e-4, atol=1e-5)
np.testing.assert_array_equal(crop_inds, actual_crop_inds)
pd.testing.assert_frame_equal(concat_windows_dataset.description,
loaded_concat_windows_dataset.description)
start_offset_samples=0,
stop_offset_samples=None,
window_size_samples=window_size_samples,
window_stride_samples=window_stride_samples,
drop_last_window=False)
# save memory by deleting raw recording
del tuh_subset
# store the number of windows required for loading later on
tuh_windows.description["n_windows"] = [
len(d) for d in tuh_windows.datasets
]
# create one directory for every recording
rec_path = os.path.join(OUT_PATH, str(rec_i))
if not os.path.exists(rec_path):
os.makedirs(rec_path)
save_concat_dataset(rec_path, tuh_windows)
out_i += 1
# save memory by catching epoched recording
del tuh_windows
else:
# store raws to disk for option of using different compute window
# sizes
pass
###############################################################################
# We load the preprocessed data again in a lazy fashion (`preload=False`). It is
# now ready to be used for model training.
tuh_loaded = load_concat_dataset('./tuh_sample/', preload=False)