def _demo_validate_real_data():
ds_rate = 2
channel_map = [1] * 16 + [0, 0, 1, 1, 0, 1, 1, 1, 0]
data_train_folder = load_experimental_data()
mode = 'calibration'
raw_dat, stamp_time, channels, type_amp, fs = read_data_csv(
data_train_folder + '/rawdata.csv')
dat = sig_pro(raw_dat, fs=fs, k=ds_rate)
# Get data and labels
s_i, t_t_i, t_i = trigger_decoder(mode=mode,
trigger_loc=data_train_folder +
'/triggers.txt')
x_train, y_train, num_seq, _ = trial_reshaper(t_t_i,
t_i,
dat,
mode=mode,
fs=fs,
k=ds_rate,
channel_map=channel_map)
model = train_pca_rda_kde_model(x_train, y_train, k_folds=10)
fig = plt.figure()
ax = fig.add_subplot(211)
x_plot = np.linspace(np.min(model.line_el[-1]), np.max(model.line_el[-1]),
1000)[:, np.newaxis]
ax.plot(model.line_el[2][y_train == 0],
-0.005 -
0.01 * np.random.random(model.line_el[2][y_train == 0].shape[0]),
'ro',
label='class(-)')
ax.plot(model.line_el[2][y_train == 1],
-0.005 -
0.01 * np.random.random(model.line_el[2][y_train == 1].shape[0]),
'go',
label='class(+)')
for idx in range(len(model.pipeline[2].list_den_est)):
log_dens = model.pipeline[2].list_den_est[idx].score_samples(x_plot)
ax.plot(x_plot[:, 0],
np.exp(log_dens),
'r-' * (idx == 0) + 'g-' * (idx == 1),
linewidth=2.0)
ax.legend(loc='upper right')
plt.title('Training Data')
plt.ylabel('p(e|l)')
plt.xlabel('scores')
# Test
data_test_folder = load_experimental_data()
mode = 'calibration'
raw_dat, stamp_time, channels, type_amp, fs = read_data_csv(
data_test_folder + '/rawdata.csv')
dat = sig_pro(raw_dat, fs=fs, k=ds_rate)
# Get data and labels
s_i, t_t_i, t_i = trigger_decoder(mode=mode,
trigger_loc=data_test_folder +
'/triggers.txt')
x_test, y_test, num_seq, _ = trial_reshaper(t_t_i,
t_i,
dat,
mode=mode,
fs=fs,
k=ds_rate,
channel_map=channel_map)
model.transform(x_test)
ax.plot(model.line_el[2][y_test == 0],
-0.01 -
0.01 * np.random.random(model.line_el[2][y_test == 0].shape[0]),
'bo',
label='t_class(-)')
ax.plot(model.line_el[2][y_test == 1],
-0.01 -
0.01 * np.random.random(model.line_el[2][y_test == 1].shape[0]),
'ko',
label='t_class(+)')
bandwidth = 1.06 * min(np.std(model.line_el[2]),
iqr(model.line_el[2]) / 1.34) * np.power(
model.line_el[2].shape[0], -0.2)
test_kde = KernelDensityEstimate(bandwidth=bandwidth)
test_kde.fit(model.line_el[2], y_test)
for idx in range(len(model.pipeline[2].list_den_est)):
log_dens = test_kde.list_den_est[idx].score_samples(x_plot)
ax.plot(x_plot[:, 0],
np.exp(log_dens),
'b--' * (idx == 0) + 'k--' * (idx == 1),
linewidth=2.0)
ax.legend(loc='upper right')
plt.title('Training Data')
plt.ylabel('p(e|l)')
plt.xlabel('scores')
plt.show()
def demo1():
# A 21 channel dummy input
inp = np.array([range(4000)] * 21)
# Make the first sample indicate the channel index
# ease of checking channel removal
inp[:, 0] = np.transpose(np.arange(1, 22, 1))
# Uncomment the calibration mode you want to test trial reshaper for below:
# demo_mode = 'calibration'
# demo_mode = 'copy_phrase'
demo_mode = 'free_spell'
# Create a mock triggers.txt according to demo_mode
sample_triggers = [
'< first_pres_target 0.670907955815', '+ fixation 2.65230878454',
'< target 3.23335159523', 'F nontarget 3.46778439358',
'K nontarget 3.70046001566', 'K nontarget 3.92969065203',
'A nontarget 4.16515404402', 'N nontarget 4.39758758984',
'K nontarget 4.62848090783', 'D nontarget 4.8619234586',
'D nontarget 5.09161170403', 'A nontarget 5.32642637414',
'Y first_pres_target 6.49467007555', '+ fixation 8.47466339368',
'Y target 9.05767254303', 'R nontarget 9.29237042196',
'G nontarget 9.52458454194', '< nontarget 9.75552882335',
'< nontarget 9.98478034058', 'B nontarget 10.2205293401',
'D nontarget 10.4523640657', 'P nontarget 10.6860699275',
'O nontarget 10.9172955694', 'N nontarget 11.1487296659',
'A first_pres_target 12.2988197721', '+ fixation 14.2818938998',
'A target 14.8640901118', 'M nontarget 15.0989079671',
'J nontarget 15.3305852016', '_ nontarget 15.562809939',
'Z nontarget 15.7947462376', 'C nontarget 16.0268616159',
'O nontarget 16.2568418393', 'L nontarget 16.4914501783',
'R nontarget 16.722291825', 'C nontarget 16.9548927715',
'L first_pres_target 18.1060283357', '+ fixation 20.0890030139',
'L target 20.6712063042', 'J nontarget 20.9039095314',
'M nontarget 21.1352675367', 'S nontarget 21.3701048572',
'U nontarget 21.6018058039', 'P nontarget 21.8331666405',
'< nontarget 22.065949852', 'D nontarget 22.2980032956',
'O nontarget 22.5301154888', 'P nontarget 22.7622121098'
]
if demo_mode == 'copy_phrase':
sample_triggers = filter(lambda x: 'first_pres_target' not in x,
sample_triggers)
if demo_mode == 'free_spell':
sample_triggers = map(
lambda x: x.replace('fixation', '').replace('target', '').replace(
'non', ''),
filter(lambda x: 'first_pres_target' not in x, sample_triggers))
with open('triggers.txt', 'w+') as text_file:
for line in sample_triggers:
text_file.write(line + '\n')
# Load trigger file
trigger_data = trigger_decoder(trigger_loc='triggers.txt', mode=demo_mode)
# Remove the trigger.txt file that has been used.
remove('triggers.txt')
# Which channels to keep//
channel_map = [1] * 21
channel_map[0] = 0 # Remove first channel
channel_map[20] = 0 # Remove 21st channel
channel_map[3:15] = [0] * 12 # remove 4th to 15th channels
# In the end there are only 7 channels left
# reshape function is applied to dummy data with given trigger file
arg = trial_reshaper(trial_target_info=trigger_data[1],
timing_info=trigger_data[2],
filtered_eeg=inp,
fs=256,
k=2,
mode=demo_mode,
channel_map=channel_map)
# Print results.
print('Reshaped trials:\n', arg[0], '\nLabels:', arg[1],
'\nTotal number of sequences:', arg[2],
'\nTrial number in each sequence:', arg[3])
def offline_analysis(data_folder: str = None,
parameters: dict = {},
alert_finished: bool = True):
""" Gets calibration data and trains the model in an offline fashion.
pickle dumps the model into a .pkl folder
Args:
data_folder(str): folder of the data
save all information and load all from this folder
parameter(dict): parameters for running offline analysis
alert_finished(bool): whether or not to alert the user offline analysis complete
How it Works:
- reads data and information from a .csv calibration file
- reads trigger information from a .txt trigger file
- filters data
- reshapes and labels the data for the training procedure
- fits the model to the data
- uses cross validation to select parameters
- based on the parameters, trains system using all the data
- pickle dumps model into .pkl file
- generates and saves offline analysis screen
- [optional] alert the user finished processing
"""
if not data_folder:
data_folder = load_experimental_data()
mode = 'calibration'
trial_length = parameters.get('collection_window_after_trial_length')
raw_dat, _, channels, type_amp, fs = read_data_csv(
data_folder + '/' + parameters.get('raw_data_name', 'raw_data.csv'))
log.info(f'Channels read from csv: {channels}')
log.info(f'Device type: {type_amp}')
downsample_rate = parameters.get('down_sampling_rate', 2)
# Remove 60hz noise with a notch filter
notch_filter_data = notch.notch_filter(raw_dat, fs, frequency_to_remove=60)
# bandpass filter from 2-45hz
filtered_data = bandpass.butter_bandpass_filter(notch_filter_data,
2,
45,
fs,
order=2)
# downsample
data = downsample.downsample(filtered_data, factor=downsample_rate)
# Process triggers.txt
triggers_file = parameters.get('trigger_file_name', 'triggers.txt')
_, t_t_i, t_i, offset = trigger_decoder(
mode=mode, trigger_path=f'{data_folder}/{triggers_file}')
static_offset = parameters.get('static_trigger_offset', 0)
offset = offset + static_offset
# Channel map can be checked from raw_data.csv file.
# read_data_csv already removes the timespamp column.
channel_map = analysis_channels(channels, type_amp)
x, y, _, _ = trial_reshaper(t_t_i,
t_i,
data,
mode=mode,
fs=fs,
k=downsample_rate,
offset=offset,
channel_map=channel_map,
trial_length=trial_length)
k_folds = parameters.get('k_folds', 10)
model, auc = train_pca_rda_kde_model(x, y, k_folds=k_folds)
log.info('Saving offline analysis plots!')
# After obtaining the model get the transformed data for plotting purposes
model.transform(x)
generate_offline_analysis_screen(
x,
y,
model=model,
folder=data_folder,
down_sample_rate=downsample_rate,
fs=fs,
save_figure=True,
show_figure=False,
channel_names=analysis_channel_names_by_pos(channels, channel_map))
log.info('Saving the model!')
with open(data_folder + f'/model_{auc}.pkl', 'wb') as output:
pickle.dump(model, output)
if alert_finished:
offline_analysis_tone = parameters.get('offline_analysis_tone')
play_sound(offline_analysis_tone)
return model
def offline_analysis(data_folder=None, parameters={}):
""" Gets calibration data and trains the model in an offline fashion.
pickle dumps the model into a .pkl folder
Args:
data_folder(str): folder of the data
save all information and load all from this folder
Duty cycle
- reads data and information from a .csv calibration file
- reads trigger information from a .txt trigger file
- filters data
- reshapes and labels the data for the training procedure
- fits the model to the data
- uses cross validation to select parameters
- based on the parameters, trains system using all the data
- pickle dumps model into .pkl file
- generates and saves offline analysis screen
"""
if not data_folder:
data_folder = load_experimental_data()
mode = 'calibration'
raw_dat, stamp_time, channels, type_amp, fs = read_data_csv(
data_folder + '/' + parameters.get('raw_data_name', 'raw_data.csv'))
print(f'Channels read from csv: {channels}')
print(f'Device type: {type_amp}')
downsample_rate = parameters.get('down_sampling_rate', 2)
filtered_data = sig_pro(raw_dat, fs=fs, k=downsample_rate)
# Process triggers.txt
triggers_file = parameters.get('triggers_file_name', 'triggers.txt')
_, t_t_i, t_i, offset = trigger_decoder(
mode=mode, trigger_loc=f"{data_folder}/{triggers_file}")
# Channel map can be checked from raw_data.csv file.
# read_data_csv already removes the timespamp column.
channel_map = analysis_channels(channels, type_amp)
x, y, num_seq, _ = trial_reshaper(t_t_i,
t_i,
filtered_data,
mode=mode,
fs=fs,
k=downsample_rate,
offset=offset,
channel_map=channel_map)
k_folds = parameters.get('k_folds', 10)
model = train_pca_rda_kde_model(x, y, k_folds=10)
print('Saving offline analysis plots!')
generate_offline_analysis_screen(x, y, model, data_folder)
print('Saving the model!')
with open(data_folder + '/model.pkl', 'wb') as output:
pickle.dump(model, output)
return model