def use_pyprep(srcPath, dstPath):
summaryFile = open("./summary.txt", 'a')
src = Path(srcPath)
dst = Path(dstPath)
session_list = [1, 4]
for file in src.glob("*.txt"):
status, uid, session, trial, task, preprocess = get_file_info(file)
# print(file)
if status:
x = 0
# if task != 'Baseline' and int(session) in session_list:
dst_file = dst / (file.with_suffix('').name[:-3] + 'pyprep.edf')
print("Processsing ", file.name)
raw, montage = load_to_mne(file)
# # #Only for debugging purposes
# raw = raw.crop(0, 15).load_data()
raw.crop(tmin=(raw.times[-1] - 120)) #Get only last two minutes
# Extract some info
eeg_index = mne.pick_types(raw.info,
eeg=True,
eog=False,
meg=False)
ch_names = raw.info["ch_names"]
ch_names_eeg = list(np.asarray(ch_names)[eeg_index])
sample_rate = raw.info["sfreq"]
# Make a copy of the data
raw_copy = raw.copy()
# Fit prep
prep_params = {
'ref_chs': ch_names_eeg,
'reref_chs': ch_names_eeg,
'line_freqs': np.arange(60, sample_rate / 2, 60)
}
prep = PrepPipeline(raw_copy, prep_params, montage)
prep.fit()
final_eeg = prep.raw
# Write pyprep summary of interpolation
summaryFile.write(file.name + "\n")
summaryFile.write(str(prep.interpolated_channels) + "\n")
summaryFile.flush()
write_data_edf(final_eeg, dst_file, uid, task)
def test_prep_pipeline_filter_kwargs(raw, montage):
"""Test prep pipeline with filter kwargs."""
eeg_index = mne.pick_types(raw.info, eeg=True, eog=False, meg=False)
raw_copy = raw.copy()
ch_names = raw_copy.info["ch_names"]
ch_names_eeg = list(np.asarray(ch_names)[eeg_index])
sample_rate = raw_copy.info["sfreq"]
prep_params = {
"ref_chs": ch_names_eeg,
"reref_chs": ch_names_eeg,
"line_freqs": np.arange(60, sample_rate / 2, 60),
}
filter_kwargs = {
"method": "fir",
"phase": "zero-double",
}
prep = PrepPipeline(
raw_copy, prep_params, montage, random_state=42, filter_kwargs=filter_kwargs
)
prep.fit()
def test_prep_pipeline_non_eeg(raw, montage):
"""Test prep pipeline with non eeg channels."""
raw_copy = raw.copy()
# make arbitrary non eeg channels from the register
sfreq = raw_copy.info["sfreq"] # Sampling frequency
times = np.array(list(range(raw_copy._data.shape[1])))
ch_names_non_eeg = ["misc" + str(i) for i in range(4)]
ch_types_non_eeg = ["misc" for i in range(4)]
raw_non_eeg, _, _ = make_random_mne_object(
ch_names_non_eeg,
ch_types_non_eeg,
times,
sfreq,
RNG=np.random.RandomState(1337),
)
raw_copy.add_channels([raw_non_eeg], force_update_info=True)
prep_params = {
"ref_chs": "eeg",
"reref_chs": "eeg",
"line_freqs": np.arange(60, sfreq / 2, 60),
}
prep = PrepPipeline(raw_copy, prep_params, montage, random_state=42)
prep.fit()
# correct non-eeg channels configured in init
assert set(prep.ch_names_non_eeg) == set(ch_names_non_eeg)
# original (all) channel names same as full_raw names
assert set(prep.raw.ch_names) == set(raw_copy.ch_names)
# names of raw (only eeg) same as full names - non eeg names
assert set(prep.raw_eeg.ch_names) == set(raw_copy.ch_names) - set(ch_names_non_eeg)
# quantity of raw (only eeg) same as quantity of all - non eeg lists
assert prep.raw_eeg._data.shape[0] == len(raw_copy.ch_names) - len(
prep.ch_names_non_eeg
)
# quantity of channels in is the same as qty of full raw
assert raw_copy._data.shape[0] == prep.raw._data.shape[0]
def perform_prep(self):
""" perform_prep
Calls pyprep's PrepPipeline and detects bad channels in the data.
Returns
-------
bad_chs : List
List of all the bad channel names
"""
self.automagic["prep"]["performed"] = True
montage = mne.channels.make_standard_montage(
self.params["interpolation_params"]["montage"])
montage.ch_names = [
'E37', 'E44', 'E86', 'E39', 'E87', 'E93', 'E60', 'E77', 'E61',
'E72', 'E75', 'E50', 'E112', 'E34', 'E2', 'E51', 'E98', 'E4',
'E42', 'E101', 'E36', 'E17', 'E88', 'E7', 'E25', 'E5', 'E18',
'E84', 'E21', 'E49', 'E103', 'E97', 'E122', 'E71', 'E53', 'E54',
'E32', 'E67', 'E38', 'E22', 'E114', 'E92', 'E108', 'E121', 'E3',
'E14', 'E13', 'E64', 'E90', 'E20', 'E27', 'E107', 'E100', 'E119',
'E47', 'E110', 'E83', 'E40', 'E117', 'E48', 'E15', 'E30', 'E26',
'E55', 'E69', 'E79', 'E125', 'E76', 'E128', 'E41', 'E43', 'E74',
'E24', 'E45', 'E9', 'E73', 'E102', 'E29', 'E65', 'E63', 'E12',
'E82', 'E127', 'E56', 'E52', 'E91', 'E80', 'E95', 'E124', 'E58',
'E31', 'E106', 'E46', 'E78', 'E6', 'E115', 'E126', 'E89', 'E96',
'E123', 'E8', 'E66', 'E104', 'E10', 'E11', 'E118', 'E57', 'E120',
'E99', 'E116', 'E16', 'E94', 'E111', 'E105', 'E19', 'E35', 'E109',
'E1', 'E33', 'E68', 'E28', 'E59', 'E70', 'E113', 'E81', 'E23',
'E85', 'E62'
]
prep = PrepPipeline(self.eeg, self.params["interpolation_params"],
montage)
prep = prep.fit()
self.bad_chs = list(
set(prep.still_noisy_channels)
| set(prep.interpolated_channels)
| set(prep.bad_before_interpolation))
self.automagic.update({"auto_bad_chans": self.bad_chs})
return self.bad_chs
def perform_prep(self):
""" perform_prep
Calls pyprep's PrepPipeline and detects bad channels in the data.
Returns
-------
bad_chs : List
List of all the bad channel names
"""
self.automagic["prep"]["performed"] = True
montage = mne.channels.make_standard_montage(
self.params["interpolation_params"]["montage"]
)
prep = PrepPipeline(self.eeg, self.params["interpolation_params"], montage)
prep = prep.fit()
self.bad_chs = list(
set(prep.still_noisy_channels)
| set(prep.interpolated_channels)
| set(prep.bad_before_interpolation)
)
self.automagic.update({"auto_bad_chans": self.bad_chs})
return self.bad_chs
def test_prep_pipeline(raw, montage):
"""Test prep pipeline."""
eeg_index = mne.pick_types(raw.info, eeg=True, eog=False, meg=False)
raw_copy = raw.copy()
ch_names = raw_copy.info["ch_names"]
ch_names_eeg = list(np.asarray(ch_names)[eeg_index])
sample_rate = raw_copy.info["sfreq"]
prep_params = {
"ref_chs": ch_names_eeg,
"reref_chs": ch_names_eeg,
"line_freqs": np.arange(60, sample_rate / 2, 60),
}
prep = PrepPipeline(raw_copy, prep_params, montage, random_state=42)
prep.fit()
EEG_raw = raw_copy.get_data(picks="eeg") * 1e6
EEG_raw_max = np.max(abs(EEG_raw), axis=None)
EEG_raw_matlab = sio.loadmat("./examples/matlab_results/EEG_raw.mat")
EEG_raw_matlab = EEG_raw_matlab["save_data"]
EEG_raw_diff = EEG_raw - EEG_raw_matlab
# EEG_raw_mse = (EEG_raw_diff / EEG_raw_max ** 2).mean(axis=None)
fig, axs = plt.subplots(5, 3, sharex="all")
plt.setp(fig, facecolor=[1, 1, 1])
fig.suptitle("Python versus Matlab PREP results", fontsize=16)
im = axs[0, 0].imshow(
EEG_raw / EEG_raw_max,
aspect="auto",
extent=[0, (EEG_raw.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[0, 0].set_title("Python", fontsize=14)
axs[0, 1].imshow(
EEG_raw_matlab / EEG_raw_max,
aspect="auto",
extent=[0, (EEG_raw_matlab.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[0, 1].set_title("Matlab", fontsize=14)
axs[0, 2].imshow(
EEG_raw_diff / EEG_raw_max,
aspect="auto",
extent=[0, (EEG_raw_diff.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[0, 2].set_title("Difference", fontsize=14)
# axs[0, 0].set_title('Original EEG', loc='left', fontsize=14)
# axs[0, 0].set_ylabel('Channel Number', fontsize=14)
cb = fig.colorbar(im, ax=axs, fraction=0.05, pad=0.04)
cb.set_label("\u03BCVolt", fontsize=14)
EEG_new_matlab = sio.loadmat("./examples/matlab_results/EEGNew.mat")
EEG_new_matlab = EEG_new_matlab["save_data"]
EEG_new = prep.EEG_new
EEG_new_max = np.max(abs(EEG_new), axis=None)
EEG_new_diff = EEG_new - EEG_new_matlab
# EEG_new_mse = ((EEG_new_diff / EEG_new_max) ** 2).mean(axis=None)
axs[1, 0].imshow(
EEG_new / EEG_new_max,
aspect="auto",
extent=[0, (EEG_new.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[1, 1].imshow(
EEG_new_matlab / EEG_new_max,
aspect="auto",
extent=[0, (EEG_new_matlab.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[1, 2].imshow(
EEG_new_diff / EEG_new_max,
aspect="auto",
extent=[0, (EEG_new_diff.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
# axs[1, 0].set_title('High pass filter', loc='left', fontsize=14)
# axs[1, 0].set_ylabel('Channel Number', fontsize=14)
EEG_clean_matlab = sio.loadmat("./examples/matlab_results/EEG.mat")
EEG_clean_matlab = EEG_clean_matlab["save_data"]
EEG_clean = prep.EEG
EEG_max = np.max(abs(EEG_clean), axis=None)
EEG_diff = EEG_clean - EEG_clean_matlab
# EEG_mse = ((EEG_diff / EEG_max) ** 2).mean(axis=None)
axs[2, 0].imshow(
EEG_clean / EEG_max,
aspect="auto",
extent=[0, (EEG_clean.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[2, 1].imshow(
EEG_clean_matlab / EEG_max,
aspect="auto",
extent=[0, (EEG_clean_matlab.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[2, 2].imshow(
EEG_diff / EEG_max,
aspect="auto",
extent=[0, (EEG_diff.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
# axs[2, 0].set_title('Line-noise removal', loc='left', fontsize=14)
axs[2, 0].set_ylabel("Channel Number", fontsize=14)
EEG = prep.EEG_before_interpolation
EEG_max = np.max(abs(EEG), axis=None)
EEG_ref_mat = sio.loadmat("./examples/matlab_results/EEGref.mat")
EEG_ref_matlab = EEG_ref_mat["save_EEG"]
# reference_matlab = EEG_ref_mat["save_reference"]
EEG_ref_diff = EEG - EEG_ref_matlab
# EEG_ref_mse = ((EEG_ref_diff / EEG_max) ** 2).mean(axis=None)
# reference_signal = prep.reference_before_interpolation
# reference_max = np.max(abs(reference_signal), axis=None)
# reference_diff = reference_signal - reference_matlab
# reference_mse = ((reference_diff / reference_max) ** 2).mean(axis=None)
axs[3, 0].imshow(
EEG / EEG_max,
aspect="auto",
extent=[0, (EEG.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[3, 1].imshow(
EEG_ref_matlab / EEG_max,
aspect="auto",
extent=[0, (EEG_ref_matlab.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[3, 2].imshow(
EEG_ref_diff / EEG_max,
aspect="auto",
extent=[0, (EEG_ref_diff.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
# axs[3, 0].set_title('Referencing', loc='left', fontsize=14)
# axs[3, 0].set_ylabel('Channel Number', fontsize=14)
EEG_final = prep.raw.get_data() * 1e6
EEG_final_max = np.max(abs(EEG_final), axis=None)
EEG_final_matlab = sio.loadmat("./examples/matlab_results/EEGinterp.mat")
EEG_final_matlab = EEG_final_matlab["save_data"]
EEG_final_diff = EEG_final - EEG_final_matlab
# EEG_final_mse = ((EEG_final_diff / EEG_final_max) ** 2).mean(axis=None)
axs[4, 0].imshow(
EEG_final / EEG_final_max,
aspect="auto",
extent=[0, (EEG_final.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[4, 1].imshow(
EEG_final_matlab / EEG_final_max,
aspect="auto",
extent=[0, (EEG_final_matlab.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
axs[4, 2].imshow(
EEG_final_diff / EEG_final_max,
aspect="auto",
extent=[0, (EEG_final_diff.shape[1] / sample_rate), 63, 0],
vmin=-1,
vmax=1,
cmap=plt.get_cmap("RdBu"),
)
# axs[4, 0].set_title('Interpolation', loc='left', fontsize=14)
# axs[4, 0].set_ylabel('Channel Number', fontsize=14)
axs[4, 1].set_xlabel("Time(s)", fontsize=14)
# Set PREP parameters and run PREP
# --------------------------------
#
# Notes: we keep all the default parameter settings as described in the PREP
# paper except one, the fraction of bad time windows
# (we change it from 0.01 to 0.1), because the EEG data is 60s long, which
# means it gots only 60 time windows. We think the algorithm will be too
# sensitive if using the default setting.
# Fit prep
prep_params = {
"ref_chs": ch_names_eeg,
"reref_chs": ch_names_eeg,
"line_freqs": np.arange(60, sample_rate / 2, 60),
}
prep = PrepPipeline(raw_copy, prep_params, montage)
prep.fit()
###############################################################################
# Results
# -------
#
# You can check the detected bad channels in each step of PREP.
print("Bad channels: {}".format(prep.interpolated_channels))
print("Bad channels original: {}".format(
prep.noisy_channels_original["bad_all"]))
print("Bad channels after interpolation: {}".format(prep.still_noisy_channels))
# Matlab's results
# ----------------
ch_names_eeg = list(np.asarray(ch_names)[eeg_index])
sample_rate = raw_copy.info["sfreq"]
###############################################################################
# Step 1: Set PREP parameters and run PREP
#
# Notes: we keep all the default parameter settings as described in [PREP paper](https://www.ncbi.nlm.nih.gov/pubmed/26150785)
# except one, the fraction of bad time windows (we change it from 0.01 to 0.1), because the EEG data is 60s long,
# which means it gots only 60 time windows. We think the algorithm will be too sensitive if using the default setting.
prep_params = {
"ref_chs": ch_names_eeg,
"reref_chs": ch_names_eeg,
"line_freqs": np.arange(60, sample_rate / 2, 60),
}
prep = PrepPipeline(raw_copy, prep_params)
prep.fit()
# check results
print("Bad channels: {}".format(prep.interpolated_channels))
print("Bad channels original: {}".format(
prep.noisy_channels_original["bad_all"]))
print("Bad channels after interpolation: {}".format(prep.still_noisy_channels))
###############################################################################
# Step 2: Validation
#
# To validate each step of pyprep's results, we compare results after each step with the results from EEGLAB's PREP
#
# To make it easy to compare, we rescale the EEG data to [-1, 1] (devided the data by maximum absolute value) when making the plot