def demo_copy_phrase_wrapper():
# We need to train a dummy model
num_x_p = 100
num_x_n = 900
x_p = mean_pos + var_pos * np.random.randn(num_ch, num_x_p, dim_x)
x_n = mean_neg + var_neg * np.random.randn(num_ch, num_x_n, dim_x)
y_p = [1] * num_x_p
y_n = [0] * num_x_n
train_x = np.concatenate((x_p, x_n), 1)
train_y = np.concatenate((y_p, y_n), 0)
permutation = np.random.permutation(train_x.shape[1])
train_x = train_x[:, permutation, :]
train_y = train_y[permutation]
train_x = train_x[list(np.where(np.asarray(channel_map) == 1)[0]), :, :]
k_folds = 10
model, _ = train_pca_rda_kde_model(train_x, train_y, k_folds=k_folds)
# Define task and operate
task_list = [('I_LOVE_COOKIES', 'I_LOVE_'),
('THIS_IS_A_DEMO', 'THIS_IS_A_')]
task = CopyPhraseWrapper(min_num_inq=1,
max_num_inq=25,
signal_model=model,
fs=dim_x * 2,
k=1,
alp=alphabet(),
task_list=task_list)
print(task)
def _demo_validate_data():
dim_x = 75
num_x_p = 500
num_x_n = 500
num_ch = 20
x_p_train = np.asarray(
[np.random.randn(num_x_p, dim_x) for i in range(num_ch)])
x_n_train = np.array(
[np.random.randn(num_x_p, dim_x) for i in range(num_ch)])
y_p_train = [1] * num_x_p
y_n_train = [0] * num_x_n
x_train = np.concatenate((x_n_train, x_p_train), axis=1)
y_train = np.concatenate((y_n_train, y_p_train), axis=0)
permutation = np.random.permutation(x_train.shape[1])
x_train = x_train[:, permutation, :]
y_train = y_train[permutation]
model, _ = train_pca_rda_kde_model(x_train, y_train, k_folds=10)
fig = plt.figure()
ax = fig.add_subplot(211)
x_plot = np.linspace(np.min(model.line_el[-1]), np.max(model.line_el[-1]),
1000)[:, np.newaxis]
ax.plot(model.line_el[2][y_train == 0],
-0.005 -
0.01 * np.random.random(model.line_el[2][y_train == 0].shape[0]),
'ro',
label='class(-)')
ax.plot(model.line_el[2][y_train == 1],
-0.005 -
0.01 * np.random.random(model.line_el[2][y_train == 1].shape[0]),
'go',
label='class(+)')
for idx in range(len(model.pipeline[2].list_den_est)):
log_dens = model.pipeline[2].list_den_est[idx].score_samples(x_plot)
ax.plot(x_plot[:, 0],
np.exp(log_dens),
'r-' * (idx == 0) + 'g-' * (idx == 1),
linewidth=2.0)
ax.legend(loc='upper right')
plt.title('Training Data')
plt.ylabel('p(e|l)')
plt.xlabel('scores')
# Test
x_p_test = np.asarray(
[np.random.randn(num_x_p, dim_x) for i in range(num_ch)])
x_n_test = np.array(
[np.random.randn(num_x_p, dim_x) for i in range(num_ch)])
y_p_test = [1] * num_x_p
y_n_test = [0] * num_x_n
x_test = np.concatenate((x_n_test, x_p_test), axis=1)
y_test = np.concatenate((y_n_test, y_p_test), axis=0)
permutation = np.random.permutation(x_test.shape[1])
x_test = x_test[:, permutation, :]
y_test = y_test[permutation]
model.transform(x_test)
ax.plot(model.line_el[2][y_test == 0],
-0.01 -
0.01 * np.random.random(model.line_el[2][y_test == 0].shape[0]),
'bo',
label='t_class(-)')
ax.plot(model.line_el[2][y_test == 1],
-0.01 -
0.01 * np.random.random(model.line_el[2][y_test == 1].shape[0]),
'ko',
label='t_class(+)')
bandwidth = 1.06 * min(np.std(model.line_el[2]),
iqr(model.line_el[2]) / 1.34) * np.power(
model.line_el[2].shape[0], -0.2)
test_kde = KernelDensityEstimate(bandwidth=bandwidth)
test_kde.fit(model.line_el[2], y_test)
for idx in range(len(model.pipeline[2].list_den_est)):
log_dens = test_kde.list_den_est[idx].score_samples(x_plot)
ax.plot(x_plot[:, 0],
np.exp(log_dens),
'b--' * (idx == 0) + 'k--' * (idx == 1),
linewidth=2.0)
ax.legend(loc='upper right')
plt.title('Training Data')
plt.ylabel('p(e|l)')
plt.xlabel('scores')
plt.show()
mean_neg = 0
var_neg = .5
x_p = mean_pos + var_pos * np.random.randn(num_ch, num_x_p, dim_x)
x_n = mean_neg + var_neg * np.random.randn(num_ch, num_x_n, dim_x)
y_p = [1] * num_x_p
y_n = [0] * num_x_n
x = np.concatenate((x_p, x_n), 1)
y = np.concatenate(np.asarray([y_p, y_n]), 0)
permutation = np.random.permutation(x.shape[1])
x = x[:, permutation, :]
y = y[permutation]
k_folds = 10
model, _ = train_pca_rda_kde_model(x, y, k_folds=k_folds)
alp = [
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
'R', 'S', 'T', 'U', 'V', 'Y', 'Z', '<', '_'
]
num_x_p = 1
num_x_n = 9
x_p_s = mean_pos + var_pos * np.random.randn(num_ch, num_x_p, dim_x)
x_n_s = mean_neg + var_neg * np.random.randn(num_ch, num_x_n, dim_x)
x_s = np.concatenate((x_n_s, x_p_s), 1)
idx_let = np.random.permutation(len(alp))
letters = [alp[i] for i in idx_let[0:(num_x_p + num_x_n)]]
def _demo_validate_real_data():
ds_rate = 2
channel_map = [1] * 16 + [0, 0, 1, 1, 0, 1, 1, 1, 0]
data_train_folder = load_experimental_data()
mode = 'calibration'
raw_dat, stamp_time, channels, type_amp, fs = read_data_csv(
data_train_folder + '/rawdata.csv')
dat = sig_pro(raw_dat, fs=fs, k=ds_rate)
# Get data and labels
s_i, t_t_i, t_i = trigger_decoder(mode=mode,
trigger_loc=data_train_folder +
'/triggers.txt')
x_train, y_train, num_seq, _ = trial_reshaper(t_t_i,
t_i,
dat,
mode=mode,
fs=fs,
k=ds_rate,
channel_map=channel_map)
model = train_pca_rda_kde_model(x_train, y_train, k_folds=10)
fig = plt.figure()
ax = fig.add_subplot(211)
x_plot = np.linspace(np.min(model.line_el[-1]), np.max(model.line_el[-1]),
1000)[:, np.newaxis]
ax.plot(model.line_el[2][y_train == 0],
-0.005 -
0.01 * np.random.random(model.line_el[2][y_train == 0].shape[0]),
'ro',
label='class(-)')
ax.plot(model.line_el[2][y_train == 1],
-0.005 -
0.01 * np.random.random(model.line_el[2][y_train == 1].shape[0]),
'go',
label='class(+)')
for idx in range(len(model.pipeline[2].list_den_est)):
log_dens = model.pipeline[2].list_den_est[idx].score_samples(x_plot)
ax.plot(x_plot[:, 0],
np.exp(log_dens),
'r-' * (idx == 0) + 'g-' * (idx == 1),
linewidth=2.0)
ax.legend(loc='upper right')
plt.title('Training Data')
plt.ylabel('p(e|l)')
plt.xlabel('scores')
# Test
data_test_folder = load_experimental_data()
mode = 'calibration'
raw_dat, stamp_time, channels, type_amp, fs = read_data_csv(
data_test_folder + '/rawdata.csv')
dat = sig_pro(raw_dat, fs=fs, k=ds_rate)
# Get data and labels
s_i, t_t_i, t_i = trigger_decoder(mode=mode,
trigger_loc=data_test_folder +
'/triggers.txt')
x_test, y_test, num_seq, _ = trial_reshaper(t_t_i,
t_i,
dat,
mode=mode,
fs=fs,
k=ds_rate,
channel_map=channel_map)
model.transform(x_test)
ax.plot(model.line_el[2][y_test == 0],
-0.01 -
0.01 * np.random.random(model.line_el[2][y_test == 0].shape[0]),
'bo',
label='t_class(-)')
ax.plot(model.line_el[2][y_test == 1],
-0.01 -
0.01 * np.random.random(model.line_el[2][y_test == 1].shape[0]),
'ko',
label='t_class(+)')
bandwidth = 1.06 * min(np.std(model.line_el[2]),
iqr(model.line_el[2]) / 1.34) * np.power(
model.line_el[2].shape[0], -0.2)
test_kde = KernelDensityEstimate(bandwidth=bandwidth)
test_kde.fit(model.line_el[2], y_test)
for idx in range(len(model.pipeline[2].list_den_est)):
log_dens = test_kde.list_den_est[idx].score_samples(x_plot)
ax.plot(x_plot[:, 0],
np.exp(log_dens),
'b--' * (idx == 0) + 'k--' * (idx == 1),
linewidth=2.0)
ax.legend(loc='upper right')
plt.title('Training Data')
plt.ylabel('p(e|l)')
plt.xlabel('scores')
plt.show()
def offline_analysis(data_folder: str = None,
parameters: dict = {},
alert_finished: bool = True):
""" Gets calibration data and trains the model in an offline fashion.
pickle dumps the model into a .pkl folder
Args:
data_folder(str): folder of the data
save all information and load all from this folder
parameter(dict): parameters for running offline analysis
alert_finished(bool): whether or not to alert the user offline analysis complete
How it Works:
- reads data and information from a .csv calibration file
- reads trigger information from a .txt trigger file
- filters data
- reshapes and labels the data for the training procedure
- fits the model to the data
- uses cross validation to select parameters
- based on the parameters, trains system using all the data
- pickle dumps model into .pkl file
- generates and saves offline analysis screen
- [optional] alert the user finished processing
"""
if not data_folder:
data_folder = load_experimental_data()
mode = 'calibration'
trial_length = parameters.get('collection_window_after_trial_length')
raw_dat, _, channels, type_amp, fs = read_data_csv(
data_folder + '/' + parameters.get('raw_data_name', 'raw_data.csv'))
log.info(f'Channels read from csv: {channels}')
log.info(f'Device type: {type_amp}')
downsample_rate = parameters.get('down_sampling_rate', 2)
# Remove 60hz noise with a notch filter
notch_filter_data = notch.notch_filter(raw_dat, fs, frequency_to_remove=60)
# bandpass filter from 2-45hz
filtered_data = bandpass.butter_bandpass_filter(notch_filter_data,
2,
45,
fs,
order=2)
# downsample
data = downsample.downsample(filtered_data, factor=downsample_rate)
# Process triggers.txt
triggers_file = parameters.get('trigger_file_name', 'triggers.txt')
_, t_t_i, t_i, offset = trigger_decoder(
mode=mode, trigger_path=f'{data_folder}/{triggers_file}')
static_offset = parameters.get('static_trigger_offset', 0)
offset = offset + static_offset
# Channel map can be checked from raw_data.csv file.
# read_data_csv already removes the timespamp column.
channel_map = analysis_channels(channels, type_amp)
x, y, _, _ = trial_reshaper(t_t_i,
t_i,
data,
mode=mode,
fs=fs,
k=downsample_rate,
offset=offset,
channel_map=channel_map,
trial_length=trial_length)
k_folds = parameters.get('k_folds', 10)
model, auc = train_pca_rda_kde_model(x, y, k_folds=k_folds)
log.info('Saving offline analysis plots!')
# After obtaining the model get the transformed data for plotting purposes
model.transform(x)
generate_offline_analysis_screen(
x,
y,
model=model,
folder=data_folder,
down_sample_rate=downsample_rate,
fs=fs,
save_figure=True,
show_figure=False,
channel_names=analysis_channel_names_by_pos(channels, channel_map))
log.info('Saving the model!')
with open(data_folder + f'/model_{auc}.pkl', 'wb') as output:
pickle.dump(model, output)
if alert_finished:
offline_analysis_tone = parameters.get('offline_analysis_tone')
play_sound(offline_analysis_tone)
return model
