def modelopt(lstmsize, lstmdropout, lstmoptim, nestimators):
lstmsize = lstmsize[0] * 10
lstmdropout = lstmdropout[0]
lstmoptim = lstmoptim[0]
lstmnepochs = 50
lstmbatchsize = 64
nestimators = nestimators[0] * 100
nfolds = 5
print lstmsize, lstmdropout, lstmoptim, nestimators
# Load the dataset
static_train = np.load("/storage/hpc_anna/GMiC/Data/ECoGmixed/fourier/train_data.npy")
dynamic_train = np.load("/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_data.npy")
static_val = np.load("/storage/hpc_anna/GMiC/Data/ECoGmixed/fourier/test_data.npy")
dynamic_val = np.load("/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_data.npy")
labels_train = np.load("/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_labels.npy")
labels_val = np.load("/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_labels.npy")
# Merge train and test
static_all = np.concatenate((static_train, static_val), axis=0)
dynamic_all = np.concatenate((dynamic_train, dynamic_val), axis=0)
labels_all = np.concatenate((labels_train, labels_val), axis=0)
nsamples = static_all.shape[0]
# prepare where to store the ratios
ratios_all_lstm = np.empty(len(labels_all))
# split indices into folds
enrich_idx_list = np.array_split(range(nsamples), nfolds)
# run CV
for fid, enrich_idx in enumerate(enrich_idx_list):
train_idx = list(set(range(nsamples)) - set(enrich_idx))
# extract predictions using LSTM on dynamic
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
ratios_all_lstm[enrich_idx] = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_all[enrich_idx])
# dataset for hybrid learning
enriched_by_lstm = np.concatenate((static_all, np.matrix(ratios_all_lstm).T), axis=1)
# (2.) k-fold cross validation to obtain accuracy
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
train_idx = list(set(range(nsamples)) - set(val_idx))
# Hybrid on features enriched by HMM (3)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(enriched_by_lstm[train_idx], labels_all[train_idx])
scores.append(rf.score(enriched_by_lstm[val_idx], labels_all[val_idx]))
print "Result: %.4f" % np.mean(scores)
return -np.mean(scores)
train_idx = list(set(range(nsamples)) - set(predict_idx))
# extract predictions using RF on static
print " Extracting predictions on static data with RF..."
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(static_all[train_idx], labels_all[train_idx])
predictions_all_rf[predict_idx] = rf.predict_log_proba(static_all[predict_idx])
predictions_all_rf[predictions_all_rf == -inf] = np.min(predictions_all_rf[predictions_all_rf != -inf])
# extract predictions using LSTM on dynamic
print " Extracting predictions on dynamic data with LSTM..."
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, dynamic_all[predict_idx])
predictions_all_lstm[predict_idx] = np.vstack((mse_pos, mse_neg)).T
ratios_all_lstm[predict_idx] = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_all[predict_idx])
#
# Prepare combined datasets for the future experiments
#
# datasets for ensemble learning
predictions_combined_rf_lstm = np.concatenate((predictions_all_rf, predictions_all_lstm), axis=1)
# datasets for hybrid learning
enriched_by_lstm = np.concatenate((static_all, np.matrix(ratios_all_lstm).T), axis=1)
# dataset to check how generative models perform if provided with static features along with dynamic
static_as_dynamic = np.zeros((static_all.shape[0], static_all.shape[1], dynamic_all.shape[2]))
for i in range(static_all.shape[0]):
# extract predictions using HMM on dynamic
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype,
dynamic_all[train_idx],
labels_all[train_idx])
ratios_all_hmm[predict_idx] = hmmcl.pos_neg_ratios(
model_pos, model_neg, dynamic_all[predict_idx])
# extract predictions using LSTM on dynamic
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg,
dynamic_all[predict_idx])
ratios_all_lstm[predict_idx] = lstmcl.pos_neg_ratios(
model_pos, model_neg, dynamic_all[predict_idx])
# datasets for hybrid learning
enriched_by_hmm = np.concatenate((static_all, np.matrix(ratios_all_hmm).T),
axis=1)
enriched_by_lstm = np.concatenate(
(static_all, np.matrix(ratios_all_lstm).T), axis=1)
# k-fold CV for performance estimation
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = {
1: [],
2: [],
3: [],
4: [],
5: [],
print "Evaluating joint model:"
print "Splitting data in two halves..."
fh_idx = np.random.choice(range(0, dynamic_train.shape[0]),
size=np.round(dynamic_train.shape[0] * 0.5, 0),
replace=False)
sh_idx = list(set(range(0, dynamic_train.shape[0])) - set(fh_idx))
fh_data = dynamic_train[fh_idx, :, :]
fh_labels = labels_train[fh_idx]
sh_data = dynamic_train[sh_idx, :, :]
sh_labels = labels_train[sh_idx]
print "Training LSTM classifier..."
lstmcl = LSTMClassifier(2000, 0.5, 'adagrad', 20)
model_pos, model_neg = lstmcl.train(fh_data, fh_labels)
print "Extracting ratios based on the LSTM model..."
sh_ratios = lstmcl.pos_neg_ratios(model_pos, model_neg, sh_data)
val_ratios = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_val)
print "Merging static features and LSTM-based ratios..."
enriched_sh_data = np.hstack(
(static_train[sh_idx, :], sh_ratios.reshape(len(sh_ratios), 1)))
enriched_val_data = np.hstack(
(static_val, val_ratios.reshape(len(val_ratios), 1)))
print "Training RF on the merged dataset..."
rf = RandomForestClassifier(n_estimators=100)
rf.fit(enriched_sh_data, sh_labels)
print "RF+LSTM with enriched features on validation set: %.4f" % rf.score(
enriched_val_data, labels_val)
#
# Generative LSTM
#
print " Extracting ratios and activations from generative LSTM..."
# train the models
lstmcl = LSTMClassifier(g_lstmsize, g_lstmdropout, g_lstmoptim,
g_lstmnepochs, g_lstmbatch)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
# extract ratios
mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg,
dynamic_all[predict_idx])
ratios_generative[predict_idx] = lstmcl.pos_neg_ratios(
model_pos, model_neg, dynamic_all[predict_idx])
# extract activations
activations_pos = lstmcl.activations(model_pos, dynamic_all[predict_idx])
activations_neg = lstmcl.activations(model_neg, dynamic_all[predict_idx])
activations_generative[predict_idx] = np.concatenate(
(activations_pos[:, -1, :], activations_neg[:, -1, :]), axis=1)
#
# Discriminative LSTM
#
print " Extracting ratios and activations from discriminative LSTM..."
# train the model
lstmcl = LSTMDiscriminative(d_lstmsize, d_lstmdropout, d_lstmoptim,
d_lstmnepochs, d_lstmbatch)
def bpic(lstmsize, lstmdropout, lstmoptim, nestimators):
lstmsize = lstmsize[0] * 10
lstmdropout = lstmdropout[0]
lstmoptim = lstmoptim[0]
lstmnepochs = 50
lstmbatchsize = 256
nestimators = nestimators[0] * 100
nfolds = 5
print lstmsize, lstmdropout, lstmoptim, nestimators
# Load the dataset
static_train = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_static.npy'
)
dynamic_train = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_dynamic.npy'
)
static_val = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_static.npy'
)
dynamic_val = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_dynamic.npy'
)
labels_train = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_labels.npy'
)
labels_val = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_labels.npy'
)
# Merge train and test
static_all = np.concatenate((static_train, static_val), axis=0)
dynamic_all = np.concatenate((dynamic_train, dynamic_val), axis=0)
labels_all = np.concatenate((labels_train, labels_val), axis=0)
nsamples = static_all.shape[0]
# prepare where to store the ratios
ratios_all_lstm = np.empty(len(labels_all))
# split indices into folds
enrich_idx_list = np.array_split(range(nsamples), nfolds)
# run CV
for fid, enrich_idx in enumerate(enrich_idx_list):
train_idx = list(set(range(nsamples)) - set(enrich_idx))
# extract predictions using LSTM on dynamic
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
ratios_all_lstm[enrich_idx] = lstmcl.pos_neg_ratios(
model_pos, model_neg, dynamic_all[enrich_idx])
# dataset for hybrid learning
enriched_by_lstm = np.concatenate(
(static_all, np.matrix(ratios_all_lstm).T), axis=1)
# (2.) k-fold cross validation to obtain accuracy
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
train_idx = list(set(range(nsamples)) - set(val_idx))
# Hybrid on features enriched by HMM (3)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(enriched_by_lstm[train_idx], labels_all[train_idx])
scores.append(rf.score(enriched_by_lstm[val_idx], labels_all[val_idx]))
print 'Result: %.4f' % np.mean(scores)
return -np.mean(scores)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(trainA_static, trainA_labels)
predictions_trainB_rf = rf.predict_log_proba(trainB_static)
predictions_trainB_rf[predictions_trainB_rf == -inf] = np.min(
predictions_trainB_rf[predictions_trainB_rf != -inf])
predictions_test_rf = rf.predict_log_proba(test_static)
predictions_test_rf[predictions_test_rf == -inf] = np.min(
predictions_test_rf[predictions_test_rf != -inf])
# extract predictions using LSTM on dynamic
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(trainA_dynamic, trainA_labels)
mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, trainB_dynamic)
predictions_trainB_lstm = np.vstack((mse_pos, mse_neg)).T
ratios_trainB_lstm = lstmcl.pos_neg_ratios(model_pos, model_neg,
trainB_dynamic)
mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, test_dynamic)
predictions_test_lstm = np.vstack((mse_pos, mse_neg)).T
ratios_test_lstm = lstmcl.pos_neg_ratios(model_pos, model_neg, test_dynamic)
#
# Prepare combined datasets for the future experiments
#
# datasets for ensemble learning
trainB_predictions_combined_rf_lstm = np.concatenate(
(predictions_trainB_rf,
ratios_trainB_lstm.reshape((ratios_trainB_lstm.shape[0], 1))),
axis=1)
test_predictions_combined_rf_lstm = np.concatenate(
(predictions_test_rf,
enriched_val_data = np.hstack((static_val, val_ratios.reshape(len(val_ratios), 1))) print "Training RF on the merged dataset..." rf = RandomForestClassifier(n_estimators=rf_estimators, n_jobs=-1) rf.fit(enriched_sh_data, sh_labels) print "RF+HMM with enriched features on validation set: %.4f" % rf.score(enriched_val_data, labels_val) # RF+LSTM print "Evaluating RF+LSTM model:" print "Training LSTM classifier..." lstmcl = LSTMClassifier(2000, 0.5, 'adagrad', lstm_nepochs) model_pos, model_neg = lstmcl.train(fh_data, fh_labels) print "Extracting ratios based on the LSTM model..." sh_ratios = lstmcl.pos_neg_ratios(model_pos, model_neg, sh_data) val_ratios = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_val) print "Merging static features and LSTM-based ratios..." enriched_sh_data = np.hstack((static_train[sh_idx, :], sh_ratios.reshape(len(sh_ratios), 1))) enriched_val_data = np.hstack((static_val, val_ratios.reshape(len(val_ratios), 1))) print "Training RF on the merged dataset..." rf = RandomForestClassifier(n_estimators=rf_estimators, n_jobs=-1) rf.fit(enriched_sh_data, sh_labels) print "RF+LSTM with enriched features on validation set: %.4f" % rf.score(enriched_val_data, labels_val)
predictions_test_rf[predictions_test_rf == -inf] = np.min(predictions_test_rf[predictions_test_rf != -inf]) # extract predictions using HMM on dynamic hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, trainA_dynamic, trainA_labels) predictions_trainB_hmm = hmmcl.predict_log_proba(model_pos, model_neg, trainB_dynamic) ratios_trainB_hmm = hmmcl.pos_neg_ratios(model_pos, model_neg, trainB_dynamic) predictions_test_hmm = hmmcl.predict_log_proba(model_pos, model_neg, test_dynamic) ratios_test_hmm = hmmcl.pos_neg_ratios(model_pos, model_neg, test_dynamic) # extract predictions using LSTM on dynamic lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(trainA_dynamic, trainA_labels) mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, trainB_dynamic) predictions_trainB_lstm = np.vstack((mse_pos, mse_neg)).T ratios_trainB_lstm = lstmcl.pos_neg_ratios(model_pos, model_neg, trainB_dynamic) mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, test_dynamic) predictions_test_lstm = np.vstack((mse_pos, mse_neg)).T ratios_test_lstm = lstmcl.pos_neg_ratios(model_pos, model_neg, test_dynamic) # # Prepare combined datasets for the future experiments # # datasets for ensemble learning trainB_predictions_combined_rf_hmm = np.concatenate((predictions_trainB_rf, ratios_trainB_hmm.reshape((ratios_trainB_hmm.shape[0], 1))), axis=1) test_predictions_combined_rf_hmm = np.concatenate((predictions_test_rf, ratios_test_hmm.reshape((ratios_test_hmm.shape[0], 1))), axis=1) trainB_predictions_combined_rf_lstm = np.concatenate((predictions_trainB_rf, ratios_trainB_lstm.reshape((ratios_trainB_lstm.shape[0], 1))), axis=1) test_predictions_combined_rf_lstm = np.concatenate((predictions_test_rf, ratios_test_lstm.reshape((ratios_test_lstm.shape[0], 1))), axis=1)
print "Current fold is %d" % fid
train_idx = list(set(range(nsamples)) - set(predict_idx))
#
# Generative LSTM
#
print " Extracting ratios and activations from generative LSTM..."
# train the models
lstmcl = LSTMClassifier(g_lstmsize, g_lstmdropout, g_lstmoptim, g_lstmnepochs, g_lstmbatch)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
# extract ratios
mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, dynamic_all[predict_idx])
ratios_generative[predict_idx] = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_all[predict_idx])
# extract activations
activations_pos = lstmcl.activations(model_pos, dynamic_all[predict_idx])
activations_neg = lstmcl.activations(model_neg, dynamic_all[predict_idx])
activations_generative[predict_idx] = np.concatenate((activations_pos[:, -1, :], activations_neg[:, -1, :]), axis=1)
#
# Discriminative LSTM
#
print " Extracting ratios and activations from discriminative LSTM..."
# train the model
lstmcl = LSTMDiscriminative(d_lstmsize, d_lstmdropout, d_lstmoptim, d_lstmnepochs, d_lstmbatch)
model = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
# prepare where to store the ratios
ratios_all_lstm = np.empty(len(labels_all))
# split indices into folds
enrich_idx_list = np.array_split(range(nsamples), nfolds)
# CV for feature enrichment
for fid, enrich_idx in enumerate(enrich_idx_list):
print "Current fold is %d / %d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(enrich_idx))
# extract ratios using LSTM on dynamic
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
ratios_all_lstm[enrich_idx] = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_all[enrich_idx])
# dataset for hybrid learning
enriched_by_lstm = np.concatenate((static_all, np.matrix(ratios_all_lstm).T), axis=1)
# CV for accuracy estimation
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
print "Current fold is %d / %d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(val_idx))
# Hybrid on features enriched by HMM (3)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(enriched_by_lstm[train_idx], labels_all[train_idx])
scores.append(rf.score(enriched_by_lstm[val_idx], labels_all[val_idx]))