def function(lstmsize, lstmdropout, lstmoptim):
lstmsize = lstmsize[0] * 10
lstmdropout = lstmdropout[0]
lstmoptim = lstmoptim[0]
lstmnepochs = 50
lstmbatchsize = 64
nfolds = 5
print("Reading data...")
dynamic_train = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_data.npy')
labels_train = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_labels.npy')
nsamples = dynamic_train.shape[0]
# k-fold cross validation to obtain accuracy
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
train_idx = list(set(range(nsamples)) - set(val_idx))
print "Current fold is %d / %d" % (fid + 1, nfolds)
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_train[train_idx], labels_train[train_idx])
scores.append(lstmcl.test(model_pos, model_neg, dynamic_train[val_idx], labels_train[val_idx]))
print 'Result: %.4f' % np.mean(scores)
return -np.mean(scores)
def function(lstmsize, lstmdropout, lstmoptim):
lstmsize = lstmsize[0] * 10
lstmdropout = lstmdropout[0]
lstmoptim = lstmoptim[0]
lstmnepochs = 50
lstmbatchsize = 64
nfolds = 5
print("Reading data...")
dynamic_train = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_data.npy')
labels_train = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_labels.npy')
nsamples = dynamic_train.shape[0]
# k-fold cross validation to obtain accuracy
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
train_idx = list(set(range(nsamples)) - set(val_idx))
print "Current fold is %d / %d" % (fid + 1, nfolds)
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_train[train_idx],
labels_train[train_idx])
scores.append(
lstmcl.test(model_pos, model_neg, dynamic_train[val_idx],
labels_train[val_idx]))
print 'Result: %.4f' % np.mean(scores)
return -np.mean(scores)
def bpic(lstmsize, lstmdropout, lstmoptim):
lstmsize = lstmsize[0] * 10
lstmdropout = lstmdropout[0]
lstmoptim = lstmoptim[0]
lstmnepochs = 50
lstmbatchsize = 64
nfolds = 5
# Load the dataset
#static_train = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_static.npy')
dynamic_train = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_dynamic.npy')
#static_val = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_static.npy')
dynamic_val = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_dynamic.npy')
labels_train = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_labels.npy')
labels_val = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_labels.npy')
# Merge train and test
#static_all = np.concatenate((static_train, static_val), axis=0)
dynamic_all = np.concatenate((dynamic_train, dynamic_val), axis=0)
labels_all = np.concatenate((labels_train, labels_val), axis=0)
nsamples = dynamic_all.shape[0]
# k-fold cross validation to obtain accuracy
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
train_idx = list(set(range(nsamples)) - set(val_idx))
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
scores.append(lstmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx]))
print 'Result: %.4f' % np.mean(scores)
return -np.mean(scores)
def bpic(lstmsize, lstmdropout, lstmoptim):
lstmsize = lstmsize[0] * 10
lstmdropout = lstmdropout[0]
lstmoptim = lstmoptim[0]
lstmnepochs = 50
lstmbatchsize = 64
nfolds = 5
# Load the dataset
#static_train = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_static.npy')
dynamic_train = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_dynamic.npy'
)
#static_val = np.load('/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_static.npy')
dynamic_val = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_dynamic.npy'
)
labels_train = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/train_labels.npy'
)
labels_val = np.load(
'/storage/hpc_anna/GMiC/Data/BPIChallenge/f1/preprocessed/test_labels.npy'
)
# Merge train and test
#static_all = np.concatenate((static_train, static_val), axis=0)
dynamic_all = np.concatenate((dynamic_train, dynamic_val), axis=0)
labels_all = np.concatenate((labels_train, labels_val), axis=0)
nsamples = dynamic_all.shape[0]
# k-fold cross validation to obtain accuracy
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = []
for fid, val_idx in enumerate(val_idx_list):
train_idx = list(set(range(nsamples)) - set(val_idx))
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
scores.append(
lstmcl.test(model_pos, model_neg, dynamic_all[val_idx],
labels_all[val_idx]))
print 'Result: %.4f' % np.mean(scores)
return -np.mean(scores)
#
# Sanity Checks
#
print "Expected performance of a lonely model is 0.75, of the joint model 1.0"
# a) static data classification
rf = RandomForestClassifier(n_estimators=100)
rf.fit(static_train, labels_train)
print "Random Forest with static features on validation set: %.4f" % rf.score(
static_val, labels_val)
# b) dynamic data classification
lstmcl = LSTMClassifier(2000, 0.5, 'adagrad', 20)
model_pos, model_neg = lstmcl.train(dynamic_train, labels_train)
print "LSTM with dynamic features on validation set: %.4f" % lstmcl.test(
model_pos, model_neg, dynamic_val, labels_val)
#
# Evaluating Joint Model
#
print ""
print "Evaluating joint model:"
print "Splitting data in two halves..."
fh_idx = np.random.choice(range(0, dynamic_train.shape[0]),
size=np.round(dynamic_train.shape[0] * 0.5, 0),
replace=False)
sh_idx = list(set(range(0, dynamic_train.shape[0])) - set(fh_idx))
fh_data = dynamic_train[fh_idx, :, :]
fh_labels = labels_train[fh_idx]
sh_data = dynamic_train[sh_idx, :, :]
sh_labels = labels_train[sh_idx]
dynamic_train = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_data.npy')
static_val = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/fourier/test_data.npy')
dynamic_val = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_data.npy')
labels_train = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_labels.npy')
labels_val = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_labels.npy')
nsamples = dynamic_train.shape[0]
# split the data into training and test
train_idx = np.random.choice(range(0, nsamples),
size=np.round(nsamples * 0.7, 0),
replace=False)
test_idx = list(set(range(0, nsamples)) - set(train_idx))
# train the model and report performance
print 'Training the model...'
lstmcl = LSTMClassifier(lstmsize,
lstmdropout,
lstmoptim,
lstmnepochs,
lstmbatch,
validation_split=0.3)
model_pos, model_neg = lstmcl.train(dynamic_train[train_idx],
labels_train[train_idx])
print 'Generative LSTM classifier on dynamic features: %.4f' % lstmcl.test(
model_pos, model_neg, dynamic_train[test_idx], labels_train[test_idx])
# Ensemble on predictions by RF and LSTM (2)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(trainB_predictions_combined_rf_lstm, trainB_labels)
scores[2].append(rf.score(test_predictions_combined_rf_lstm, test_labels))
# Hybrid on features enriched by LSTM (4)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(trainB_enriched_by_lstm, trainB_labels)
scores[4].append(rf.score(test_enriched_by_lstm, test_labels))
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(trainB_dynamic, trainB_labels)
scores[8].append(lstmcl.test(model_pos, model_neg, test_dynamic, test_labels))
# LSTM on dynamic and static (turned into fake sequences) (10)
#lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
#model_pos, model_neg = lstmcl.train(trainB_dynamic_and_static_as_dynamic, trainB_labels)
#scores[10].append(lstmcl.test(model_pos, model_neg, test_dynamic_and_static_as_dynamic, test_labels))
# RF on static and dynamic (spatialized) features (11)
#rf = RandomForestClassifier(n_estimators=nestimators)
#rf.fit(trainB_static_and_dynamic_as_static, trainB_labels_all)
#scores[11].append(rf.score(test_static_and_dynamic_as_static, test_labels_all))
print "===> (2) Ensemble (RF) on predictions by RF and LSTM: %.4f (+/- %.4f) %s" % (
np.mean(scores[2]), np.std(scores[2]), scores[2])
print "===> (4) Hybrid (RF) on features enriched by LSTM: %.4f (+/- %.4f) %s" % (
np.mean(scores[4]), np.std(scores[4]), scores[4])
dynamic_as_static[test_idx], labels_all[test_idx])
# HMM on dynamic features (7)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype,
dynamic_all[train_idx],
labels_all[train_idx])
print "===> (7) HMM on dynamic features: %.4f" % hmmcl.test(
model_pos, model_neg, dynamic_all[test_idx], labels_all[test_idx])
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
print "===> (8) LSTM on dynamic features: %.4f" % lstmcl.test(
model_pos, model_neg, dynamic_all[test_idx], labels_all[test_idx])
# HMM on dynamic and static (turned into fake sequences) (9)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype,
dynamic_and_static_as_dynamic[train_idx],
labels_all[train_idx])
print "===> (9) HMM on dynamic and static features: %.4f" % hmmcl.test(
model_pos, model_neg, dynamic_and_static_as_dynamic[test_idx],
labels_all[test_idx])
# LSTM on dynamic and static (turned into fake sequences) (10)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_and_static_as_dynamic[train_idx],
labels_all[train_idx])
import numpy as np
from LSTM.lstm_classifier import LSTMClassifier
# parameters
lstmsize = 512
lstmdropout = 0.0
lstmoptim = 'rmsprop'
lstmnepochs = 50
lstmbatch = 32
# load the dataset
print 'Loading the dataset..'
static_train = np.load('/storage/hpc_anna/GMiC/Data/syn_lstm_wins/train_static.npy')
dynamic_train = np.load('/storage/hpc_anna/GMiC/Data/syn_lstm_wins/train_dynamic.npy')
static_val = np.load('/storage/hpc_anna/GMiC/Data/syn_lstm_wins/test_static.npy')
dynamic_val = np.load('/storage/hpc_anna/GMiC/Data/syn_lstm_wins/test_dynamic.npy')
labels_train = np.load('/storage/hpc_anna/GMiC/Data/syn_lstm_wins/train_labels.npy')
labels_val = np.load('/storage/hpc_anna/GMiC/Data/syn_lstm_wins/test_labels.npy')
nsamples = dynamic_train.shape[0]
# split the data into training and test
train_idx = np.random.choice(range(0, nsamples), size=np.round(nsamples * 0.7, 0), replace=False)
test_idx = list(set(range(0, nsamples)) - set(train_idx))
# train the model and report performance
print 'Training the model...'
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatch)
model_pos, model_neg = lstmcl.train(dynamic_train[train_idx], labels_train[train_idx])
print 'Generative LSTM classifier on dynamic features: %.4f' % lstmcl.test(model_pos, model_neg, dynamic_train[test_idx], labels_train[test_idx])
labels_train = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_labels.npy')
labels_val = np.load(
'/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_labels.npy')
nsamples = dynamic_train.shape[0]
# split indices into folds
val_idx_list = np.array_split(range(nsamples), nfolds)
# run CV
scores = []
for fid, val_idx in enumerate(val_idx_list):
print "Current fold is %d/%d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(val_idx))
# train the model and report performance
lstmcl = LSTMClassifier(lstmsize,
lstmdropout,
lstmoptim,
lstmnepochs,
lstmbatch,
validation_split=0.3)
model_pos, model_neg = lstmcl.train(dynamic_train[train_idx],
labels_train[train_idx])
scores.append(
lstmcl.test(model_pos, model_neg, dynamic_train[val_idx],
labels_train[val_idx]))
print 'Generative LSTM classifier on dynamic features: %.4f (+- %.4f) %s' % (
np.mean(scores), np.std(scores), scores)
# RF on dynamic features (6)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(dynamic_as_static[train_idx], labels_all[train_idx])
scores[6].append(rf.score(dynamic_as_static[val_idx], labels_all[val_idx]))
# HMM on dynamic features (7)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype,
dynamic_all[train_idx],
labels_all[train_idx])
acc, auc = hmmcl.test(model_pos, model_neg, dynamic_all[val_idx],
labels_all[val_idx])
scores[7].append(acc)
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
scores[8].append(
lstmcl.test(model_pos, model_neg, dynamic_all[val_idx],
labels_all[val_idx]))
print "===> (6) RF on dynamic (spatialized) features: %.4f (+/- %.4f) %s" % (
np.mean(scores[6]), np.std(scores[6]), scores[6])
print "===> (7) HMM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(
scores[7]), np.std(scores[7]), scores[7])
print "===> (8) LSTM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(
scores[8]), np.std(scores[8]), scores[8])
print "===> (5) RF on static features: %.4f" % rf.score(static_all[test_idx], labels_all[test_idx]) # RF on dynamic features (6) rf = RandomForestClassifier(n_estimators=nestimators) rf.fit(dynamic_as_static[train_idx], labels_all[train_idx]) print "===> (6) RF on dynamic (spatialized) features: %.4f" % rf.score(dynamic_as_static[test_idx], labels_all[test_idx]) # HMM on dynamic features (7) hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, dynamic_all[train_idx], labels_all[train_idx]) print "===> (7) HMM on dynamic features: %.4f" % hmmcl.test(model_pos, model_neg, dynamic_all[test_idx], labels_all[test_idx]) # LSTM on dynamic features (8) lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs) model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) print "===> (8) LSTM on dynamic features: %.4f" % lstmcl.test(model_pos, model_neg, dynamic_all[test_idx], labels_all[test_idx]) # HMM on dynamic and static (turned into fake sequences) (9) hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, dynamic_and_static_as_dynamic[train_idx], labels_all[train_idx]) print "===> (9) HMM on dynamic and static features: %.4f" % hmmcl.test(model_pos, model_neg, dynamic_and_static_as_dynamic[test_idx], labels_all[test_idx]) # LSTM on dynamic and static (turned into fake sequences) (10) lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs) model_pos, model_neg = lstmcl.train(dynamic_and_static_as_dynamic[train_idx], labels_all[train_idx]) print "===> (10) LSTM on dynamic features: %.4f" % lstmcl.test(model_pos, model_neg, dynamic_and_static_as_dynamic[test_idx], labels_all[test_idx]) # RF on static and dynamic (spatialized) features (11) rf = RandomForestClassifier(n_estimators=nestimators) rf.fit(static_and_dynamic_as_static[train_idx], labels_all[train_idx]) print "===> (11) RF on dynamic (spatialized) features: %.4f" % rf.score(static_and_dynamic_as_static[test_idx], labels_all[test_idx])
hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(3, 10, dynamic_train, labels_train) print "HMM with dynamic features on validation set: %.4f" % hmmcl.test(model_pos, model_neg, dynamic_val, labels_val) # dynamic data with RF print "Training RF on the dynamic dataset..." dynamic_as_static_train = dynamic_train.reshape((dynamic_train.shape[0], dynamic_train.shape[1] * dynamic_train.shape[2])) dynamic_as_static_val = dynamic_val.reshape((dynamic_val.shape[0], dynamic_val.shape[1] * dynamic_val.shape[2])) rf = RandomForestClassifier(n_estimators=rf_estimators, n_jobs=-1) rf.fit(dynamic_as_static_train, labels_train) print "RF with dynamic features on validation set: %.4f" % rf.score(dynamic_as_static_val, labels_val) # dynamic data with LSTM lstmcl = LSTMClassifier(2000, 0.5, 'adagrad', lstm_nepochs) model_pos, model_neg = lstmcl.train(dynamic_train, labels_train) print "LSTM with dynamic features on validation set: %.4f" % lstmcl.test(model_pos, model_neg, dynamic_val, labels_val) # joint models print "" print "Splitting data in two halves..." fh_idx = np.random.choice(range(0, dynamic_train.shape[0]), size=np.round(dynamic_train.shape[0] * 0.5, 0), replace=False) sh_idx = list(set(range(0, dynamic_train.shape[0])) - set(fh_idx)) fh_data = dynamic_train[fh_idx, :, :] fh_labels = labels_train[fh_idx] sh_data = dynamic_train[sh_idx, :, :] sh_labels = labels_train[sh_idx] # RF+HMM print "Evaluating RF+HMM model:" print "Training HMM classifier..."
#
# k-fold CV for performance estimation
#
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = {1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: [], 10: [], 11: []}
for fid, val_idx in enumerate(val_idx_list):
print "Current fold is %d / %d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(val_idx))
# RF on dynamic features (6)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(dynamic_as_static[train_idx], labels_all[train_idx])
scores[6].append(rf.score(dynamic_as_static[val_idx], labels_all[val_idx]))
# HMM on dynamic features (7)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, dynamic_all[train_idx], labels_all[train_idx])
acc, auc = hmmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx])
scores[7].append(acc)
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
scores[8].append(lstmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx]))
print "===> (6) RF on dynamic (spatialized) features: %.4f (+/- %.4f) %s" % (np.mean(scores[6]), np.std(scores[6]), scores[6])
print "===> (7) HMM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(scores[7]), np.std(scores[7]), scores[7])
print "===> (8) LSTM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(scores[8]), np.std(scores[8]), scores[8])
rf.fit(predictions_combined_rf_lstm[train_idx], labels_all[train_idx])
scores[2].append(
rf.score(predictions_combined_rf_lstm[val_idx], labels_all[val_idx]))
# Hybrid on features enriched by LSTM (4)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(enriched_by_lstm[train_idx], labels_all[train_idx])
scores[4].append(rf.score(enriched_by_lstm[val_idx], labels_all[val_idx]))
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx],
labels_all[train_idx])
scores[8].append(
lstmcl.test(model_pos, model_neg, dynamic_all[val_idx],
labels_all[val_idx]))
# LSTM on dynamic and static (turned into fake sequences) (10)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs,
lstmbatchsize)
model_pos, model_neg = lstmcl.train(
dynamic_and_static_as_dynamic[train_idx], labels_all[train_idx])
scores[10].append(
lstmcl.test(model_pos, model_neg,
dynamic_and_static_as_dynamic[val_idx],
labels_all[val_idx]))
print "===> (2) Ensemble (RF) on predictions by RF and LSTM: %.4f (+/- %.4f) %s" % (
np.mean(scores[2]), np.std(scores[2]), scores[2])
print "===> (4) Hybrid (RF) on features enriched by LSTM: %.4f (+/- %.4f) %s" % (
np.mean(scores[4]), np.std(scores[4]), scores[4])
# RF on dynamic features (6) rf = RandomForestClassifier(n_estimators=nestimators) rf.fit(trainB_dynamic_as_static, trainB_labels) scores[6].append(rf.score(test_dynamic_as_static, test_labels)) # HMM on dynamic features (7) hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, trainB_dynamic, trainB_labels) acc, auc = hmmcl.test(model_pos, model_neg, test_dynamic, test_labels) scores[7].append(acc) # LSTM on dynamic features (8) lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(trainB_dynamic, trainB_labels) scores[8].append(lstmcl.test(model_pos, model_neg, test_dynamic, test_labels)) # HMM on dynamic and static (turned into fake sequences) (9) #hmmcl = HMMClassifier() #model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, trainB_dynamic_and_static_as_dynamic, trainB_labels_all) #acc, auc = hmmcl.test(model_pos, model_neg, test_dynamic_and_static_as_dynamic, test_labels) #scores[9].append(acc) # LSTM on dynamic and static (turned into fake sequences) (10) #lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) #model_pos, model_neg = lstmcl.train(trainB_dynamic_and_static_as_dynamic, trainB_labels) #scores[10].append(lstmcl.test(model_pos, model_neg, test_dynamic_and_static_as_dynamic, test_labels)) # RF on static and dynamic (spatialized) features (11) #rf = RandomForestClassifier(n_estimators=nestimators) #rf.fit(trainB_static_and_dynamic_as_static, trainB_labels_all)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(dynamic_as_static[train_idx], labels_all[train_idx])
scores[6].append(rf.score(dynamic_as_static[val_idx], labels_all[val_idx]))
print "===> (6) RF on dynamic (spatialized) features: %.4f" % scores[6][-1]
# HMM on dynamic features (7)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, dynamic_all[train_idx], labels_all[train_idx])
acc, auc = hmmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx])
scores[7].append(acc)
print "===> (7) HMM on dynamic features: %.4f" % scores[7][-1]
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
scores[8].append(lstmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx]))
print "===> (8) LSTM on dynamic features: %.4f" % scores[8][-1]
# HMM on dynamic and static (turned into fake sequences) (9)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, dynamic_and_static_as_dynamic[train_idx], labels_all[train_idx])
acc, auc = hmmcl.test(model_pos, model_neg, dynamic_and_static_as_dynamic[val_idx], labels_all[val_idx])
scores[9].append(acc)
print "===> (9) HMM on dynamic and static features: %.4f" % scores[9][-1]
# LSTM on dynamic and static (turned into fake sequences) (10)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_and_static_as_dynamic[train_idx], labels_all[train_idx])
scores[10].append(lstmcl.test(model_pos, model_neg, dynamic_and_static_as_dynamic[val_idx], labels_all[val_idx]))
print "===> (10) LSTM on dynamic features: %.4f" % scores[10][-1]
val_idx_list = np.array_split(range(nsamples), nfolds)
scores = {1: [], 2: [], 3: [], 4: [], 5: [], 6: [], 7: [], 8: [], 9: [], 10: [], 11: []}
for fid, val_idx in enumerate(val_idx_list):
print "Current fold is %d / %d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(val_idx))
# RF on dynamic features (6)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(dynamic_as_static[train_idx], labels_all[train_idx])
scores[6].append(rf.score(dynamic_as_static[val_idx], labels_all[val_idx]))
# HMM on dynamic features (7)
hmmcl = HMMClassifier()
model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, dynamic_all[train_idx], labels_all[train_idx])
acc, auc = hmmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx])
scores[7].append(acc)
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
scores[8].append(lstmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx]))
print "===> (6) RF on dynamic (spatialized) features: %.4f (+/- %.4f) %s" % (
np.mean(scores[6]),
np.std(scores[6]),
scores[6],
)
print "===> (7) HMM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(scores[7]), np.std(scores[7]), scores[7])
print "===> (8) LSTM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(scores[8]), np.std(scores[8]), scores[8])
lstmoptim = 'adadelta'
lstmnepochs = 20
lstmbatch = 64
# load the dataset
print 'Loading the dataset..'
static_train = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/fourier/train_data.npy')
dynamic_train = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_data.npy')
static_val = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/fourier/test_data.npy')
dynamic_val = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_data.npy')
labels_train = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/train_labels.npy')
labels_val = np.load('/storage/hpc_anna/GMiC/Data/ECoGmixed/preprocessed/test_labels.npy')
nsamples = dynamic_train.shape[0]
# split indices into folds
val_idx_list = np.array_split(range(nsamples), nfolds)
# run CV
scores = []
for fid, val_idx in enumerate(val_idx_list):
print "Current fold is %d/%d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(val_idx))
# train the model and report performance
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatch, validation_split=0.3)
model_pos, model_neg = lstmcl.train(dynamic_train[train_idx], labels_train[train_idx])
scores.append(lstmcl.test(model_pos, model_neg, dynamic_train[val_idx], labels_train[val_idx]))
print 'Generative LSTM classifier on dynamic features: %.4f (+- %.4f) %s' % (np.mean(scores), np.std(scores), scores)
for fid, val_idx in enumerate(val_idx_list):
print "Current fold is %d / %d" % (fid + 1, nfolds)
train_idx = list(set(range(nsamples)) - set(val_idx))
# Ensemble on predictions by RF and LSTM (2)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(predictions_combined_rf_lstm[train_idx], labels_all[train_idx])
scores[2].append(rf.score(predictions_combined_rf_lstm[val_idx], labels_all[val_idx]))
# Hybrid on features enriched by LSTM (4)
rf = RandomForestClassifier(n_estimators=nestimators)
rf.fit(enriched_by_lstm[train_idx], labels_all[train_idx])
scores[4].append(rf.score(enriched_by_lstm[val_idx], labels_all[val_idx]))
# LSTM on dynamic features (8)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx])
scores[8].append(lstmcl.test(model_pos, model_neg, dynamic_all[val_idx], labels_all[val_idx]))
# LSTM on dynamic and static (turned into fake sequences) (10)
lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize)
model_pos, model_neg = lstmcl.train(dynamic_and_static_as_dynamic[train_idx], labels_all[train_idx])
scores[10].append(lstmcl.test(model_pos, model_neg, dynamic_and_static_as_dynamic[val_idx], labels_all[val_idx]))
print "===> (2) Ensemble (RF) on predictions by RF and LSTM: %.4f (+/- %.4f) %s" % (np.mean(scores[2]), np.std(scores[2]), scores[2])
print "===> (4) Hybrid (RF) on features enriched by LSTM: %.4f (+/- %.4f) %s" % (np.mean(scores[4]), np.std(scores[4]), scores[4])
print "===> (8) LSTM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(scores[8]), np.std(scores[8]), scores[8])
print "===> (10) LSTM on dynamic features: %.4f (+/- %.4f) %s" % (np.mean(scores[10]), np.std(scores[10]), scores[10])
# RF on dynamic features (6) rf = RandomForestClassifier(n_estimators=nestimators) rf.fit(trainB_dynamic_as_static, trainB_labels) scores[6] = rf.score(test_dynamic_as_static, test_labels) # HMM on dynamic features (7) hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, trainB_dynamic, trainB_labels) acc, auc = hmmcl.test(model_pos, model_neg, test_dynamic, test_labels) scores[7] = acc # LSTM on dynamic features (8) lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(trainB_dynamic, trainB_labels) scores[8] = lstmcl.test(model_pos, model_neg, test_dynamic, test_labels) # HMM on dynamic and static (turned into fake sequences) (9) hmmcl = HMMClassifier() model_pos, model_neg = hmmcl.train(nhmmstates, nhmmiter, hmmcovtype, trainB_dynamic_and_static_as_dynamic, trainB_labels) acc, auc = hmmcl.test(model_pos, model_neg, test_dynamic_and_static_as_dynamic, test_labels) scores[9] = acc # LSTM on dynamic and static (turned into fake sequences) (10) lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(trainB_dynamic_and_static_as_dynamic, trainB_labels) scores[10] = lstmcl.test(model_pos, model_neg, test_dynamic_and_static_as_dynamic, test_labels) # RF on static and dynamic (spatialized) features (11) rf = RandomForestClassifier(n_estimators=nestimators) rf.fit(trainB_static_and_dynamic_as_static, trainB_labels)
