print " Extracting ratios and activations from generative LSTM..." # train the models lstmcl = LSTMClassifier(g_lstmsize, g_lstmdropout, g_lstmoptim, g_lstmnepochs, g_lstmbatch) model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) # extract ratios mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, dynamic_all[predict_idx]) ratios_generative[predict_idx] = lstmcl.pos_neg_ratios( model_pos, model_neg, dynamic_all[predict_idx]) # extract activations activations_pos = lstmcl.activations(model_pos, dynamic_all[predict_idx]) activations_neg = lstmcl.activations(model_neg, dynamic_all[predict_idx]) activations_generative[predict_idx] = np.concatenate( (activations_pos[:, -1, :], activations_neg[:, -1, :]), axis=1) # # Discriminative LSTM # print " Extracting ratios and activations from discriminative LSTM..." # train the model lstmcl = LSTMDiscriminative(d_lstmsize, d_lstmdropout, d_lstmoptim, d_lstmnepochs, d_lstmbatch) model = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) # extract ratios
trainA_dynamic = train_dynamic[:train_half] trainB_dynamic = train_dynamic[train_half:] trainA_labels = train_labels[:train_half] trainB_labels = train_labels[train_half:] # # Train enrichment models on trainA # print 'Training enrichment models...' # train LSTM activations extractor lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(trainA_dynamic, trainA_labels) trainB_activations_pos = lstmcl.activations(model_pos, trainB_dynamic) trainB_activations_neg = lstmcl.activations(model_neg, trainB_dynamic) trainB_activations = np.concatenate((trainB_activations_pos[:, 499, :], trainB_activations_neg[:, 499, :]), axis=1) test_activations_pos = lstmcl.activations(model_pos, test_dynamic) test_activations_neg = lstmcl.activations(model_neg, test_dynamic) test_activations = np.concatenate((test_activations_pos[:, 499, :], test_activations_neg[:, 499, :]), axis=1) # # Prepare combined datasets for the future experiments # # datasets for hybrid learning trainB_enriched_by_lstm = np.concatenate((trainB_static, trainB_activations), axis=1) test_enriched_by_lstm = np.concatenate((test_static, test_activations), axis=1)
activations_all = np.empty((len(labels_all), lstmsize * 2)) # split indices into folds enrich_idx_list = np.array_split(range(nsamples), nfolds) # CV for feature enrichment for fid, enrich_idx in enumerate(enrich_idx_list): print "Current fold is %d / %d" % (fid + 1, nfolds) train_idx = list(set(range(nsamples)) - set(enrich_idx)) # train models for enrichment lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) # extract activations activations_pos = lstmcl.activations(model_pos, dynamic_all[enrich_idx]) activations_neg = lstmcl.activations(model_neg, dynamic_all[enrich_idx]) activations_all[enrich_idx] = np.concatenate((activations_pos[:, -1, :], activations_neg[:, -1, :]), axis=1) # dataset for hybrid learning enriched_by_lstm = np.concatenate((static_all, activations_all), axis=1) print static_all.shape print enriched_by_lstm.shape # CV for accuracy estimation val_idx_list = np.array_split(range(nsamples), nfolds) scores = [] for fid, val_idx in enumerate(val_idx_list): print "Current fold is %d / %d" % (fid + 1, nfolds) train_idx = list(set(range(nsamples)) - set(val_idx))
activations_all = np.empty((len(labels_all), lstmsize * 2)) # split indices into folds enrich_idx_list = np.array_split(range(nsamples), nfolds) # CV for feature enrichment for fid, enrich_idx in enumerate(enrich_idx_list): print "Enrichment fold %d / %d" % (fid + 1, nfolds) train_idx = list(set(range(nsamples)) - set(enrich_idx)) # train models for enrichment lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) # extract activations activations_pos = lstmcl.activations(model_pos, dynamic_all[enrich_idx]) activations_neg = lstmcl.activations(model_neg, dynamic_all[enrich_idx]) activations_all[enrich_idx] = np.concatenate((activations_pos[:, -1, :], activations_neg[:, -1, :]), axis=1) # dataset for hybrid learning enriched_by_lstm = np.concatenate((static_all, activations_all), axis=1) print static_all.shape print enriched_by_lstm.shape # CV for accuracy estimation val_idx_list = np.array_split(range(nsamples), nfolds) scores = [] for fid, val_idx in enumerate(val_idx_list): print "Evaluation fold %d / %d" % (fid + 1, nfolds) train_idx = list(set(range(nsamples)) - set(val_idx))
trainA_dynamic = train_dynamic[:train_half] trainB_dynamic = train_dynamic[train_half:] trainA_labels = train_labels[:train_half] trainB_labels = train_labels[train_half:] # # Train enrichment models on trainA # print 'Training enrichment models...' # train LSTM activations extractor lstmcl = LSTMClassifier(lstmsize, lstmdropout, lstmoptim, lstmnepochs, lstmbatchsize) model_pos, model_neg = lstmcl.train(trainA_dynamic, trainA_labels) trainB_activations_pos = lstmcl.activations(model_pos, trainB_dynamic) trainB_activations_neg = lstmcl.activations(model_neg, trainB_dynamic) trainB_activations = np.concatenate((trainB_activations_pos[:, seqlen - 1, :], trainB_activations_neg[:, seqlen - 1, :]), axis=1) test_activations_pos = lstmcl.activations(model_pos, test_dynamic) test_activations_neg = lstmcl.activations(model_neg, test_dynamic) test_activations = np.concatenate((test_activations_pos[:, seqlen - 1, :], test_activations_neg[:, seqlen - 1, :]), axis=1) # # Prepare combined datasets for the future experiments #
# # Generative LSTM # print " Extracting ratios and activations from generative LSTM..." # train the models lstmcl = LSTMClassifier(g_lstmsize, g_lstmdropout, g_lstmoptim, g_lstmnepochs, g_lstmbatch) model_pos, model_neg = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) # extract ratios mse_pos, mse_neg = lstmcl.predict_mse(model_pos, model_neg, dynamic_all[predict_idx]) ratios_generative[predict_idx] = lstmcl.pos_neg_ratios(model_pos, model_neg, dynamic_all[predict_idx]) # extract activations activations_pos = lstmcl.activations(model_pos, dynamic_all[predict_idx]) activations_neg = lstmcl.activations(model_neg, dynamic_all[predict_idx]) activations_generative[predict_idx] = np.concatenate((activations_pos[:, -1, :], activations_neg[:, -1, :]), axis=1) # # Discriminative LSTM # print " Extracting ratios and activations from discriminative LSTM..." # train the model lstmcl = LSTMDiscriminative(d_lstmsize, d_lstmdropout, d_lstmoptim, d_lstmnepochs, d_lstmbatch) model = lstmcl.train(dynamic_all[train_idx], labels_all[train_idx]) # extract ratios ratios_discriminative[predict_idx] = lstmcl.pos_neg_ratios(model, dynamic_all[predict_idx])