def remove_features(folds, feature_indices_list): new_folds = list() bool_feature_mask = np.ones(260) for seg in feature_indices_list: deb = seg[0] fin = seg[1] bool_feature_mask[deb:fin] = 0 bool_feature_mask = bool_feature_mask == 1 for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) X_train = X_train[:,:,bool_feature_mask] X_test = X_test[:,:,bool_feature_mask] print X_train.shape, X_test.shape data = dict() data['train'] = dict() data['train']['X'] = X_train data['train']['y'] = y_train data['train']['song_id'] = id_train data['test'] = dict() data['test']['X'] = X_test data['test']['y'] = y_test data['test']['song_id'] = id_test new_folds.append(data) return new_folds
def remove_features(folds, feature_indices_list, NUM_OUTPUT): new_folds = list() bool_feature_mask = np.ones(260) for seg in feature_indices_list: deb = seg[0] fin = seg[1] bool_feature_mask[deb:fin] = 0 bool_feature_mask = bool_feature_mask == 1 for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] X_train, y_train, id_train = load_X_from_fold_to_3dtensor( fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor( fold, 'test', NUM_OUTPUT) X_train = X_train[:, :, bool_feature_mask] X_test = X_test[:, :, bool_feature_mask] print X_train.shape, X_test.shape data = dict() data['train'] = dict() data['train']['X'] = X_train data['train']['y'] = y_train data['train']['song_id'] = id_train data['test'] = dict() data['test']['X'] = X_test data['test']['y'] = y_test data['test']['song_id'] = id_test new_folds.append(data) return new_folds
def add_essentia_features(folds, essentia_folds, feature_indices_list): new_folds = list() for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) essentia_fold = essentia_folds[fold_id] essentia_X_train = essentia_fold['train']['X'] essentia_X_test = essentia_fold['test']['X'] for seg in feature_indices_list: deb = seg[0] fin = seg[1] X_train = np.concatenate((X_train, essentia_X_train[:,:,deb:fin]), axis=2) X_test = np.concatenate((X_test, essentia_X_test[:,:,deb:fin]), axis=2) print X_train.shape data = dict() data['train'] = dict() data['train']['X'] = X_train data['train']['y'] = y_train data['train']['song_id'] = id_train data['test'] = dict() data['test']['X'] = X_test data['test']['y'] = y_test data['test']['song_id'] = id_test new_folds.append(data) return new_folds
def add_prediction_features(folds, train_pred_folds, test_pred_folds, feature_indices_list): new_folds = list() for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] X_train, y_train, id_train = load_X_from_fold_to_3dtensor( fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor( fold, 'test', NUM_OUTPUT) train_pred_fold = train_pred_folds[fold_id] test_pred_fold = test_pred_folds[fold_id] for seg in feature_indices_list: deb = seg[0] fin = seg[1] X_train = np.concatenate((X_train, train_pred_fold[:, :, deb:fin]), axis=2) X_test = np.concatenate((X_test, test_pred_fold[:, :, deb:fin]), axis=2) print X_train.shape data = dict() data['train'] = dict() data['train']['X'] = X_train data['train']['y'] = y_train data['train']['song_id'] = id_train data['test'] = dict() data['test']['X'] = X_test data['test']['y'] = y_test data['test']['song_id'] = id_test new_folds.append(data) return new_folds
def rnn_cv( folds, n_hidden=10, n_epochs=50, lr=0.001, lrd = 0.999, reg_coef= 0.01, doSmoothing=False, useEssentia=False): doSaveModel = False if doSmoothing: dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g_smoothed'%(nb_features, n_hidden, n_epochs, lr, reg_coef) else: dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g'%(nb_features, n_hidden, n_epochs, lr, reg_coef) MODELDIR = 'rnn/' + dir_name + '/' LOGDIR = MODELDIR if not path.exists(MODELDIR): makedirs(MODELDIR) print '... output dir: %s'%(MODELDIR) # smoothing params taille = 12 wts = np.ones(taille-1)*1./taille wts = np.hstack((np.array([1./(2*taille)]), wts, np.array([1./(2*taille)]))) delay = (wts.shape[0]-1) / 2 # # initialize global logger variable # print '... initializing global logger variable' # logger = logging.getLogger(__name__) # withFile = False # logger = settings.init(MODELDIR + 'train.log', withFile) # perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef) perf_file_name = LOGDIR + 'performance.log' log_f = open(perf_file_name, 'w') all_fold_pred = list() all_fold_y_test = list() all_fold_id_test = list() for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] t0 = time.time() # print '... loading FOLD %d'%fold_id # if useEssentia: # fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed_essentia.pkl'%(fold_id), "rb" ) ) if useEssentia: X_train = fold['train']['X'] y_train = fold['train']['y'] id_train = fold['train']['song_id'] X_test = fold['test']['X'] y_test = fold['test']['y'] id_test = fold['test']['song_id'] else: fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) ) X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) print X_train.shape, y_train.shape, X_test.shape, y_test.shape if useMelodyFeatures: # first feature = slope, other two = mean, std melody_train, melody_test = subset_features(all_song_melody_features, id_train, id_test) # melody_train = melody_train[:,:,1:] # melody_test = melody_test[:,:,1:] # standardize train data melody_concat_train = np.reshape(melody_train, (melody_train.shape[0]*melody_train.shape[1], melody_train.shape[2]), order='C') melody_concat_train_normed, scaler = standardize(melody_concat_train) # print concat_train_normed.shape melody_train_normed = np.reshape(melody_concat_train_normed, (melody_train.shape[0], melody_train.shape[1], melody_train.shape[2]), order='C') del melody_concat_train, melody_concat_train_normed # standardize test data melody_concat_test = np.reshape(melody_test, (melody_test.shape[0]*melody_test.shape[1], melody_test.shape[2]), order='C') melody_concat_test_normed, _ = standardize(melody_concat_test, scaler) # print concat_test_normed.shape melody_test_normed = np.reshape(melody_concat_test_normed, (melody_test.shape[0], melody_test.shape[1], melody_test.shape[2]), order='C') del melody_concat_test, melody_concat_test_normed # concat with the other features X_train = np.concatenate((X_train, melody_train_normed), axis=2) X_test = np.concatenate((X_test, melody_test_normed), axis=2) if useTempoFeatures: tempo_train, tempo_test = subset_features(all_song_tempo_features, id_train, id_test) # standardize train data tempo_concat_train = np.reshape(tempo_train, (tempo_train.shape[0]*tempo_train.shape[1], tempo_train.shape[2]), order='C') tempo_concat_train_normed, scaler = standardize(tempo_concat_train) # print concat_train_normed.shape tempo_train_normed = np.reshape(tempo_concat_train_normed, (tempo_train.shape[0], tempo_train.shape[1], tempo_train.shape[2]), order='C') del tempo_concat_train, tempo_concat_train_normed # standardize test data tempo_concat_test = np.reshape(tempo_test, (tempo_test.shape[0]*tempo_test.shape[1], tempo_test.shape[2]), order='C') tempo_concat_test_normed, _ = standardize(tempo_concat_test, scaler) # print concat_test_normed.shape tempo_test_normed = np.reshape(tempo_concat_test_normed, (tempo_test.shape[0], tempo_test.shape[1], tempo_test.shape[2]), order='C') del tempo_concat_test, tempo_concat_test_normed # concat with the other features X_train = np.concatenate((X_train, tempo_train_normed), axis=2) X_test = np.concatenate((X_test, tempo_test_normed), axis=2) # print id_test.shape # X_train = X_train[0:100,:,:] # y_train = y_train[0:100,:,:] # X_train = X_train[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]] # X_test = X_test[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]] # X_train = X_train[:,[13,85,103,142,214]] # X_test = X_test[:,[13,85,103,142,214]] # X_test = X_train[119:119+y_test.shape[0],:] # y_test = y_train[119:119+y_test.shape[0]] print X_train.shape, y_train.shape, X_test.shape, y_test.shape nb_seq_train, nb_frames_train, nb_features_train = X_train.shape nb_seq_test, nb_frames_test, nb_features_test = X_test.shape assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST' assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST' dim_ouput_train = y_train.shape[2] dim_ouput_test = y_test.shape[2] assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST' n_in = nb_features_train n_out = dim_ouput_train n_steps = nb_frames_train validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, learning_rate=lr, learning_rate_decay=lrd, L1_reg=reg_coef, L2_reg=reg_coef, n_epochs=n_epochs, activation='tanh') model.fit(X_train, y_train, validation_frequency=validation_frequency) if doSaveModel: # model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef) model_name = MODELDIR + 'model_fold%d.pkl'%(fold_id) model.save(fpath=model_name) pred = list() for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) y_hat = np.array(pred, dtype=float) print y_hat.shape if doSmoothing: # smooooooth y_hat_smooth = np.zeros_like(y_hat, dtype=float) for i in xrange(y_hat.shape[0]): y_hat_smooth[i, :, 0] = np.convolve(y_hat[i, :, 0], wts, mode='same') y_hat_smooth[i, :delay, 0] = y_hat[i, :delay, 0] y_hat_smooth[i, -delay:, 0] = y_hat[i, -delay:, 0] y_hat_smooth[i, :, 1] = np.convolve(y_hat[i, :, 1], wts, mode='same') y_hat_smooth[i, :delay, 1] = y_hat[i, :delay, 1] y_hat_smooth[i, -delay:, 1] = y_hat[i, -delay:, 1] # save predictions on the test subset, before reshaping to 2-d arrays (I need 3d arrays) if doSmoothing: # fold_pred = [item for sublist in fold_pred for item in sublist] # fold_pred = np.array(fold_pred, dtype=float) pred_file = LOGDIR + 'fold%d_test_predictions.pkl'%(fold_id) pickle.dump( y_hat_smooth, open( pred_file, "wb" ) ) print ' ... predictions y_hat_smooth saved in: %s'%(pred_file) else: # fold_pred = [item for sublist in fold_pred for item in sublist] # fold_pred = np.array(fold_pred, dtype=float) pred_file = LOGDIR + 'fold%d_test_predictions.pkl'%(fold_id) pickle.dump( y_hat, open( pred_file, "wb" ) ) print ' ... predictions y_hat saved in: %s'%(pred_file) if doSmoothing: y_hat_smooth = np.reshape(y_hat_smooth, (y_hat_smooth.shape[0]*y_hat_smooth.shape[1], y_hat_smooth.shape[2])) y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' # concat hyp labels: if doSmoothing: all_fold_pred.append(y_hat_smooth.tolist()) else: all_fold_pred.append(y_hat.tolist()) # concat ref labels: all_fold_y_test.append(y_test_concat.tolist()) if doSmoothing: RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat_smooth, id_test.shape[0]) else: RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0]) s = ( 'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n' % (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]) ) print s log_f.write(s) # predict on the train set and save predictions (useful to train rnn2) if doSmoothing: pred = list() for ind_seq_train in xrange(nb_seq_train): pred.append(model.predict(X_train[ind_seq_train])) train_y_hat = np.array(pred, dtype=float) print train_y_hat.shape train_y_hat_smooth = np.zeros_like(train_y_hat, dtype=float) for i in xrange(train_y_hat.shape[0]): train_y_hat_smooth[i, :, 0] = np.convolve(train_y_hat[i, :, 0], wts, mode='same') train_y_hat_smooth[i, :delay, 0] = train_y_hat[i, :delay, 0] train_y_hat_smooth[i, -delay:, 0] = train_y_hat[i, -delay:, 0] train_y_hat_smooth[i, :, 1] = np.convolve(train_y_hat[i, :, 1], wts, mode='same') train_y_hat_smooth[i, :delay, 1] = train_y_hat[i, :delay, 1] train_y_hat_smooth[i, -delay:, 1] = train_y_hat[i, -delay:, 1] # no reshape, I need 3d arrays # train_y_hat_smooth = np.reshape(train_y_hat_smooth, (train_y_hat_smooth.shape[0]*train_y_hat_smooth.shape[1], train_y_hat_smooth.shape[2])) pred_file = LOGDIR + 'fold%d_train_predictions.pkl'%(fold_id) pickle.dump( train_y_hat_smooth, open( pred_file, "wb" ) ) print ' ... predictions y_hat_smooth saved in: %s'%(pred_file) else: pred = list() for ind_seq_train in xrange(nb_seq_train): pred.append(model.predict(X_train[ind_seq_train])) train_y_hat = np.array(pred, dtype=float) pred_file = LOGDIR + 'fold%d_train_predictions.pkl'%(fold_id) pickle.dump( train_y_hat, open( pred_file, "wb" ) ) print ' ... predictions y_hat saved in: %s'%(pred_file) doPlot = False if doPlot: fig, ax = plt.subplots() x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0]) if EMO == 'valence': ax.plot(x1, y_test_concat[:, 0], 'o', label="Data") # ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction") ax.plot(x1, y_hat[:,0], 'ro', label="OLS prediction") else: ax.plot(x1, y_test_concat[:, 1], 'o', label="Data") ax.plot(x1, y_hat[:,1], 'ro', label="OLS prediction") plt.title(EMO + ' on Test subset') ax.legend(loc="best") plt.show() # plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png') doPlotTrain = False if doPlotTrain: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_train[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_train[0]) guess = model.predict(X_train[0]) guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() doPlotTest = False if doPlotTest: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_test[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_test[0]) # guess = model.predict(X_test[0]) guess = y_hat[0] guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() print "... Elapsed time: %f" % (time.time() - t0) all_fold_pred = [item for sublist in all_fold_pred for item in sublist] all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist] all_fold_pred = np.array(all_fold_pred, dtype=float) all_fold_y_test = np.array(all_fold_y_test, dtype=float) print all_fold_pred.shape, all_fold_y_test.shape # save predictions pred_file = LOGDIR + 'all_predictions.pkl' pickle.dump( all_fold_pred, open( pred_file, "wb" ) ) print ' ... all predictions saved in: %s'%(pred_file) # ref_file = 'rnn/all_groundtruth.pkl' # pickle.dump( all_fold_y_test, open( ref_file, "wb" ) ) # compute t-test p-values with baseline predictions baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl' baseline_preds = pickle.load(open( baseline_prediction_file, 'r' )) pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1] pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1] pvalues = (pvalue_val, pvalue_ar) RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0) # print( # 'sklearn --> valence: %.4f, arousal: %.4f\n' # 'Pearson Corr --> valence: %.4f, arousal: %.4f \n' # # % (RMSE[0], -1. , pcorr[0][0], -1) # % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0]) # ) s = ( 'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar) ) print s log_f.write(s) log_f.close() return RMSE, pcorr, pvalues
# load baseline folds folds = load_folds(DATADIR) # test using predictions as features: ajout_log_file_name = 'ajout_predictions_as_features_after_smoothing.log' ajout_log_file = open(ajout_log_file_name, 'a') feature_indices_list = [[0, 2]] train_pred_folds, test_pred_folds = load_prediction_folds( 'rnn/nfeat%d_nh10_ne50_lr0.001_reg0.01_smoothed/' % (nb_features)) new_folds = list() for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] _, y_train, id_train = load_X_from_fold_to_3dtensor( fold, 'train', NUM_OUTPUT) _, y_test, id_test = load_X_from_fold_to_3dtensor( fold, 'test', NUM_OUTPUT) X_train = train_pred_folds[fold_id] X_test = test_pred_folds[fold_id] data = dict() data['train'] = dict() data['train']['X'] = X_train data['train']['y'] = y_train data['train']['song_id'] = id_train data['test'] = dict() data['test']['X'] = X_test data['test']['y'] = y_test data['test']['song_id'] = id_test
# DATADIR = './train/' nb_features = 260 from sklearn.decomposition import PCA, KernelPCA # pca = PCA(n_components=0.95, whiten='False') pca = KernelPCA(kernel="rbf", eigen_solver = 'auto') max_nb_samples = 10000 # number of samples to fit the kernel PCA model, if bigger then 1e+4 -> too big output dim for fold_id in range(1,10): # fold_id = 0 t0 = time.time() print '... loading FOLD %d'%fold_id fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) ) X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) X_concat_train = np.reshape(X_train, (X_train.shape[0]*X_train.shape[1], X_train.shape[2]), order='C') X_concat_test = np.reshape(X_test, (X_test.shape[0]*X_test.shape[1], X_test.shape[2]), order='C') np.random.seed(321) perm = np.random.permutation(X_concat_train.shape[0]) subset_ind = perm[0:max_nb_samples] X_concat_train_SUBSET = X_concat_train[subset_ind] start_time = time.time() pca_model = pca.fit(X_concat_train_SUBSET) print("--- kPCA fitting: %.2f seconds ---" % (time.time() - start_time)) start_time = time.time()
fold = pickle.load( open( nom, "rb" ) ) fold_res = dict() for subset in ['train', 'test']: print ' ... creating subset: ', subset # doDuplicates = True # X, y, song_ids, genre_indexes = load_data_from_fold_to_3dtensor_and_genre_info(fold, subset, NUM_OUTPUT, NUM_GENRES) # # print X.shape, y.shape, genre_indexes.shape # # (572, 60, 260) (572, 60, 2) (15, 2) # # instead of (because of duplicates): # # (387, 60, 260) (387, 60, 2) (15, 2) # # # fold_res[subset] = dict() # fold_res[subset]['X'] = X # fold_res[subset]['y'] = y # fold_res[subset]['genre'] = genre_indexes doDuplicates = False X, y, song_ids = load_X_from_fold_to_3dtensor(fold, subset, NUM_OUTPUT) # print X.shape, y.shape, song_ids.shape # (387, 60, 260) (387, 60, 2) () fold_res[subset] = dict() fold_res[subset]['X'] = X fold_res[subset]['y'] = y fold_res[subset]['song_ids'] = song_ids write_fold_to_mat_files(DATADIR, fold_res, fold_id, doNormalize, doDuplicates)
def train_lstm( dim_proj=260, # word embeding dimension and LSTM number of hidden units. patience=10, # Number of epoch to wait before early stop if no progress max_epochs=5000, # The maximum number of epoch to run dispFreq=10, # Display to stdout the training progress every N updates decay_c=0., # Weight decay for the classifier applied to the U weights. lrate=0.0001, # Learning rate for sgd (not used for adadelta and rmsprop) n_words=10000, # Vocabulary size optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate). encoder='lstm', # TODO: can be removed must be lstm. saveto='lstm_model.npz', # The best model will be saved there validFreq=370, # Compute the validation error after this number of update. saveFreq=1110, # Save the parameters after every saveFreq updates maxlen=100, # Sequence longer then this get ignored batch_size=16, # The batch size during training. valid_batch_size=64, # The batch size used for validation/test set. fold_id=1, n_ex = None, dim_output=0, # Parameter for extra option noise_std=0., use_dropout=True, # if False slightly faster, but worst test error # This frequently need a bigger model. reload_model=None, # Path to a saved model we want to start from. test_size=-1, # If >0, we keep only this number of test example. ): # Model options model_options = locals().copy() print "LSTM EMO --> model options", model_options print 'Loading data' # train, valid, test = load_data(n_words=n_words, valid_portion=0.05, # maxlen=maxlen) best_ep = 0 NUM_OUTPUT = 2 # A CHANGER SI SUR MON MAC DATADIR = '/baie/corpus/emoMusic/train/' # DATADIR = '..' print '... loading FOLD %d'%fold_id fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) ) X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) # train, valid, test = (X_train, y_train) , (X_test, y_test) , (X_test, y_test) if n_ex == None: ## test only on one dimension 0 = Valence 1 = Arousal train, valid, test = (X_train, y_train[:,:,dim_output]) , (X_test, y_test[:,:,dim_output]) , (X_test, y_test[:,:,dim_output]) else: ## test only 30 values ## test only on one dimension train, valid, test = (X_train[0:n_ex,:,:], y_train[0:n_ex,:,0]) , (X_test[:,:,:], y_test[:,:,0]) , (X_test[:,:,:], y_test[:,:,0]) ## test both dimension # train, valid, test = (X_train[0:n_ex,:,:], y_train[0:n_ex,:,:]) , (X_test[:,:,:], y_test[:,:,:]) , (X_test[:,:,:], y_test[:,:,:]) ##################### A VOIR ############################################# # if test_size > 0: # # The test set is sorted by size, but we want to keep random # # size example. So we must select a random selection of the # # examples. # idx = numpy.arange(len(test[0])) # numpy.random.shuffle(idx) # idx = idx[:test_size] # test = ([test[0][n] for n in idx], [test[1][n] for n in idx]) ########################################################################### ##################### NO YDIM SINCE LINEAR REGRESSION ######################## # ydim = numpy.max(train[1]) + 1 # # model_options['ydim'] = ydim print 'Building model' # This create the initial parameters as numpy ndarrays. # Dict name (string) -> numpy ndarray params = init_params(model_options) if reload_model: load_params('lstm_model.npz', params) # This create Theano Shared Variable from the parameters. # Dict name (string) -> Theano Tensor Shared Variable # params and tparams have different copy of the weights. tparams = init_tparams(params) # use_noise is for dropout # (use_noise, x, mask, # y, f_pred_prob, cost) = build_model(tparams, model_options) (use_noise, x, y, f_pred_prob, cost) = build_model(tparams, model_options) # x0 = tensor.matrix('x0', dtype=config.floatX) # (use_noise, x, mask, # y, f_pred_prob, f_pred, cost) = build_model(x0, tparams, model_options) if decay_c > 0.: decay_c = theano.shared(numpy_floatX(decay_c), name='decay_c') weight_decay = 0. weight_decay += (tparams['U'] ** 2).sum() weight_decay *= decay_c cost += weight_decay # f_cost = theano.function([x, mask, y], cost, name='f_cost') f_cost = theano.function([x, y], cost, name='f_cost') grads = tensor.grad(cost, wrt=tparams.values()) # f_grad = theano.function([x, mask, y], grads, name='f_grad') f_grad = theano.function([x, y], grads, name='f_grad') lr = tensor.scalar(name='lr') # f_grad_shared, f_update = optimizer(lr, tparams, grads, # x, mask, y, cost) f_grad_shared, f_update = optimizer(lr, tparams, grads, x, y, cost) print 'Optimization' # shuffle index for the test and validation sets kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size) kf_test = get_minibatches_idx(len(test[0]), valid_batch_size) print "%d train examples" % len(train[0]) print "%d valid examples" % len(valid[0]) print "%d test examples" % len(test[0]) history_errs = [] tot_cost = [] best_p = None bad_count = 0 if validFreq == -1: validFreq = len(train[0]) / batch_size if saveFreq == -1: saveFreq = len(train[0]) / batch_size uidx = 0 # the number of update done estop = False # early stop start_time = time.time() try: for eidx in xrange(max_epochs): n_samples = 0 # Get new shuffled index for the training set. kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True) for _, train_index in kf: uidx += 1 # use_noise.set_value(1.) use_noise.set_value(0.) # Select the random examples for this minibatch # y = [train[1][t] for t in train_index] # x = [train[0][t]for t in train_index] # Get the data in numpy.ndarray format # This swap the axis! # Return something of shape (minibatch maxlen, n samples) # x, mask, y = prepare_data(x, y) # mask = numpy.ones((dim_proj,1)).astype(theano.config.floatX) # mask = numpy.ones((1,dim_proj)).astype(theano.config.floatX) # x = train[0][train_index].astype('int64') x = train[0][train_index].transpose(1,0,2).astype(config.floatX) # x = numpy.array([train[0][t,:,:] for t in train_index] # x = X_train y = train[1][train_index].astype(theano.config.floatX) # y = train[1][train_index,:].astype(theano.config.floatX) # y = y_train[train_index].astype(theano.config.floatX) # cost = f_grad_shared(x, mask, y, allow_input_downcast=True) n_samples += x.shape[1] # print x.shape # cost = f_grad_shared(x, mask, y) # theano.printing.debugprint(f_grad_shared) cost = f_grad_shared(x.transpose(1,0,2), y) tot_cost.append(cost) f_update(lrate) if numpy.isnan(cost) or numpy.isinf(cost): print 'NaN detected' return 1., 1., 1. if numpy.mod(uidx, dispFreq) == 0: # print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost print 'Epoch ', eidx, 'Update ', uidx, 'tot_Cost ', numpy.mean(tot_cost) tot_cost = [] if saveto and numpy.mod(uidx, saveFreq) == 0: print 'Saving...', if best_p is not None: params = best_p else: params = unzip(tparams) numpy.savez(saveto, history_errs=history_errs, **params) pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) print 'Done' if numpy.mod(uidx, validFreq) == 0: use_noise.set_value(0.) # train_err = pred_error(f_pred, prepare_data, train, kf) # valid_err = pred_error(f_pred, prepare_data, valid, # kf_valid) # test_err = pred_error(f_pred, prepare_data, test, kf_test) # train_err = cost train_err = pred_loss(f_pred_prob, train, kf) # f_pred_prob([]) valid_err = pred_loss(f_pred_prob, valid, kf_valid) test_err = pred_loss(f_pred_prob, test, kf_test) history_errs.append([valid_err, test_err]) if (uidx == 0 or valid_err <= numpy.array(history_errs)[:, 0].min()): best_p = unzip(tparams) bad_counter = 0 best_ep = eidx # print ('Train ', train_err.flatten()[0], 'Valid ', valid_err, # 'Test ', test_err) print ('Train ', train_err, 'Valid ', valid_err, 'Test ', test_err) if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, 0].min()): bad_counter += 1 if bad_counter > patience: print 'Early Stop!' estop = True break print 'Seen %d samples' % n_samples if estop: break except KeyboardInterrupt: print "Training interupted" end_time = time.time() if best_p is not None: zipp(best_p, tparams) else:# best_p = unzip(tparams) use_noise.set_value(0.) kf_train_sorted = get_minibatches_idx(len(train[0]), batch_size) # train_err = pred_error(f_pred, prepare_data, train, kf_train_sorted) # valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) # test_err = pred_error(f_pred, prepare_data, test, kf_test) train_err, tr_corr = pred_loss(f_pred_prob, train, kf_train_sorted, pearsonr=pearsonr) valid_err, val_corr = pred_loss(f_pred_prob, valid, kf_valid, pearsonr=pearsonr) test_err = pred_loss(f_pred_prob, test, kf_test) print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err if saveto: numpy.savez(saveto, train_err=train_err, valid_err=valid_err, test_err=test_err, history_errs=history_errs, **best_p) print 'The code run for %d epochs, with %f sec/epochs' % ( (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))) print >> sys.stderr, ('Training took %.1fs' % (end_time - start_time)) return train_err, valid_err, test_err, tr_corr, val_corr, best_ep
def rnn_cv(folds, n_hidden=10, n_epochs=50, lr=0.001, lrd=0.999, reg_coef=0.01, doSmoothing=False, useEssentia=False): doSaveModel = False if doSmoothing: dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g_smoothed' % ( nb_features, n_hidden, n_epochs, lr, reg_coef) else: dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g' % (nb_features, n_hidden, n_epochs, lr, reg_coef) MODELDIR = 'rnn/' + dir_name + '/' LOGDIR = MODELDIR if not path.exists(MODELDIR): makedirs(MODELDIR) print '... output dir: %s' % (MODELDIR) # smoothing params taille = 12 wts = np.ones(taille - 1) * 1. / taille wts = np.hstack( (np.array([1. / (2 * taille)]), wts, np.array([1. / (2 * taille)]))) delay = (wts.shape[0] - 1) / 2 # # initialize global logger variable # print '... initializing global logger variable' # logger = logging.getLogger(__name__) # withFile = False # logger = settings.init(MODELDIR + 'train.log', withFile) # perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef) perf_file_name = LOGDIR + 'performance.log' log_f = open(perf_file_name, 'w') all_fold_pred = list() all_fold_y_test = list() all_fold_id_test = list() for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] t0 = time.time() # print '... loading FOLD %d'%fold_id # if useEssentia: # fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed_essentia.pkl'%(fold_id), "rb" ) ) if useEssentia: X_train = fold['train']['X'] y_train = fold['train']['y'] id_train = fold['train']['song_id'] X_test = fold['test']['X'] y_test = fold['test']['y'] id_test = fold['test']['song_id'] else: fold = pickle.load( open(DATADIR + '/pkl/fold%d_normed.pkl' % (fold_id), "rb")) X_train, y_train, id_train = load_X_from_fold_to_3dtensor( fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor( fold, 'test', NUM_OUTPUT) print X_train.shape, y_train.shape, X_test.shape, y_test.shape if useMelodyFeatures: # first feature = slope, other two = mean, std melody_train, melody_test = subset_features( all_song_melody_features, id_train, id_test) # melody_train = melody_train[:,:,1:] # melody_test = melody_test[:,:,1:] # standardize train data melody_concat_train = np.reshape( melody_train, (melody_train.shape[0] * melody_train.shape[1], melody_train.shape[2]), order='C') melody_concat_train_normed, scaler = standardize( melody_concat_train) # print concat_train_normed.shape melody_train_normed = np.reshape( melody_concat_train_normed, (melody_train.shape[0], melody_train.shape[1], melody_train.shape[2]), order='C') del melody_concat_train, melody_concat_train_normed # standardize test data melody_concat_test = np.reshape( melody_test, (melody_test.shape[0] * melody_test.shape[1], melody_test.shape[2]), order='C') melody_concat_test_normed, _ = standardize(melody_concat_test, scaler) # print concat_test_normed.shape melody_test_normed = np.reshape( melody_concat_test_normed, (melody_test.shape[0], melody_test.shape[1], melody_test.shape[2]), order='C') del melody_concat_test, melody_concat_test_normed # concat with the other features X_train = np.concatenate((X_train, melody_train_normed), axis=2) X_test = np.concatenate((X_test, melody_test_normed), axis=2) if useTempoFeatures: tempo_train, tempo_test = subset_features(all_song_tempo_features, id_train, id_test) # standardize train data tempo_concat_train = np.reshape( tempo_train, (tempo_train.shape[0] * tempo_train.shape[1], tempo_train.shape[2]), order='C') tempo_concat_train_normed, scaler = standardize(tempo_concat_train) # print concat_train_normed.shape tempo_train_normed = np.reshape( tempo_concat_train_normed, (tempo_train.shape[0], tempo_train.shape[1], tempo_train.shape[2]), order='C') del tempo_concat_train, tempo_concat_train_normed # standardize test data tempo_concat_test = np.reshape( tempo_test, (tempo_test.shape[0] * tempo_test.shape[1], tempo_test.shape[2]), order='C') tempo_concat_test_normed, _ = standardize(tempo_concat_test, scaler) # print concat_test_normed.shape tempo_test_normed = np.reshape( tempo_concat_test_normed, (tempo_test.shape[0], tempo_test.shape[1], tempo_test.shape[2]), order='C') del tempo_concat_test, tempo_concat_test_normed # concat with the other features X_train = np.concatenate((X_train, tempo_train_normed), axis=2) X_test = np.concatenate((X_test, tempo_test_normed), axis=2) # print id_test.shape # X_train = X_train[0:100,:,:] # y_train = y_train[0:100,:,:] # X_train = X_train[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]] # X_test = X_test[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]] # X_train = X_train[:,[13,85,103,142,214]] # X_test = X_test[:,[13,85,103,142,214]] # X_test = X_train[119:119+y_test.shape[0],:] # y_test = y_train[119:119+y_test.shape[0]] print X_train.shape, y_train.shape, X_test.shape, y_test.shape nb_seq_train, nb_frames_train, nb_features_train = X_train.shape nb_seq_test, nb_frames_test, nb_features_test = X_test.shape assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST' assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST' dim_ouput_train = y_train.shape[2] dim_ouput_test = y_test.shape[2] assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST' n_in = nb_features_train n_out = dim_ouput_train n_steps = nb_frames_train validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, learning_rate=lr, learning_rate_decay=lrd, L1_reg=reg_coef, L2_reg=reg_coef, n_epochs=n_epochs, activation='tanh') model.fit(X_train, y_train, validation_frequency=validation_frequency) if doSaveModel: # model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef) model_name = MODELDIR + 'model_fold%d.pkl' % (fold_id) model.save(fpath=model_name) pred = list() for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) y_hat = np.array(pred, dtype=float) print y_hat.shape if doSmoothing: # smooooooth y_hat_smooth = np.zeros_like(y_hat, dtype=float) for i in xrange(y_hat.shape[0]): y_hat_smooth[i, :, 0] = np.convolve(y_hat[i, :, 0], wts, mode='same') y_hat_smooth[i, :delay, 0] = y_hat[i, :delay, 0] y_hat_smooth[i, -delay:, 0] = y_hat[i, -delay:, 0] y_hat_smooth[i, :, 1] = np.convolve(y_hat[i, :, 1], wts, mode='same') y_hat_smooth[i, :delay, 1] = y_hat[i, :delay, 1] y_hat_smooth[i, -delay:, 1] = y_hat[i, -delay:, 1] # save predictions on the test subset, before reshaping to 2-d arrays (I need 3d arrays) if doSmoothing: # fold_pred = [item for sublist in fold_pred for item in sublist] # fold_pred = np.array(fold_pred, dtype=float) pred_file = LOGDIR + 'fold%d_test_predictions.pkl' % (fold_id) pickle.dump(y_hat_smooth, open(pred_file, "wb")) print ' ... predictions y_hat_smooth saved in: %s' % (pred_file) else: # fold_pred = [item for sublist in fold_pred for item in sublist] # fold_pred = np.array(fold_pred, dtype=float) pred_file = LOGDIR + 'fold%d_test_predictions.pkl' % (fold_id) pickle.dump(y_hat, open(pred_file, "wb")) print ' ... predictions y_hat saved in: %s' % (pred_file) if doSmoothing: y_hat_smooth = np.reshape( y_hat_smooth, (y_hat_smooth.shape[0] * y_hat_smooth.shape[1], y_hat_smooth.shape[2])) y_hat = np.reshape(y_hat, (y_hat.shape[0] * y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape( y_test, (y_test.shape[0] * y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' # concat hyp labels: if doSmoothing: all_fold_pred.append(y_hat_smooth.tolist()) else: all_fold_pred.append(y_hat.tolist()) # concat ref labels: all_fold_y_test.append(y_test_concat.tolist()) if doSmoothing: RMSE, pcorr, error_per_song, mean_per_song = evaluate( y_test_concat, y_hat_smooth, id_test.shape[0]) else: RMSE, pcorr, error_per_song, mean_per_song = evaluate( y_test_concat, y_hat, id_test.shape[0]) s = ('fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n' % (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])) print s log_f.write(s) # predict on the train set and save predictions (useful to train rnn2) if doSmoothing: pred = list() for ind_seq_train in xrange(nb_seq_train): pred.append(model.predict(X_train[ind_seq_train])) train_y_hat = np.array(pred, dtype=float) print train_y_hat.shape train_y_hat_smooth = np.zeros_like(train_y_hat, dtype=float) for i in xrange(train_y_hat.shape[0]): train_y_hat_smooth[i, :, 0] = np.convolve(train_y_hat[i, :, 0], wts, mode='same') train_y_hat_smooth[i, :delay, 0] = train_y_hat[i, :delay, 0] train_y_hat_smooth[i, -delay:, 0] = train_y_hat[i, -delay:, 0] train_y_hat_smooth[i, :, 1] = np.convolve(train_y_hat[i, :, 1], wts, mode='same') train_y_hat_smooth[i, :delay, 1] = train_y_hat[i, :delay, 1] train_y_hat_smooth[i, -delay:, 1] = train_y_hat[i, -delay:, 1] # no reshape, I need 3d arrays # train_y_hat_smooth = np.reshape(train_y_hat_smooth, (train_y_hat_smooth.shape[0]*train_y_hat_smooth.shape[1], train_y_hat_smooth.shape[2])) pred_file = LOGDIR + 'fold%d_train_predictions.pkl' % (fold_id) pickle.dump(train_y_hat_smooth, open(pred_file, "wb")) print ' ... predictions y_hat_smooth saved in: %s' % (pred_file) else: pred = list() for ind_seq_train in xrange(nb_seq_train): pred.append(model.predict(X_train[ind_seq_train])) train_y_hat = np.array(pred, dtype=float) pred_file = LOGDIR + 'fold%d_train_predictions.pkl' % (fold_id) pickle.dump(train_y_hat, open(pred_file, "wb")) print ' ... predictions y_hat saved in: %s' % (pred_file) doPlot = False if doPlot: fig, ax = plt.subplots() x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0]) if EMO == 'valence': ax.plot(x1, y_test_concat[:, 0], 'o', label="Data") # ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction") ax.plot(x1, y_hat[:, 0], 'ro', label="OLS prediction") else: ax.plot(x1, y_test_concat[:, 1], 'o', label="Data") ax.plot(x1, y_hat[:, 1], 'ro', label="OLS prediction") plt.title(EMO + ' on Test subset') ax.legend(loc="best") plt.show() # plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png') doPlotTrain = False if doPlotTrain: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_train[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_train[0]) guess = model.predict(X_train[0]) guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() doPlotTest = False if doPlotTest: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_test[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_test[0]) # guess = model.predict(X_test[0]) guess = y_hat[0] guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() print "... Elapsed time: %f" % (time.time() - t0) all_fold_pred = [item for sublist in all_fold_pred for item in sublist] all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist] all_fold_pred = np.array(all_fold_pred, dtype=float) all_fold_y_test = np.array(all_fold_y_test, dtype=float) print all_fold_pred.shape, all_fold_y_test.shape # save predictions pred_file = LOGDIR + 'all_predictions.pkl' pickle.dump(all_fold_pred, open(pred_file, "wb")) print ' ... all predictions saved in: %s' % (pred_file) # ref_file = 'rnn/all_groundtruth.pkl' # pickle.dump( all_fold_y_test, open( ref_file, "wb" ) ) # compute t-test p-values with baseline predictions baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl' baseline_preds = pickle.load(open(baseline_prediction_file, 'r')) pvalue_val = stats.ttest_ind(baseline_preds[:, 0], all_fold_pred[:, 0])[1] pvalue_ar = stats.ttest_ind(baseline_preds[:, 1], all_fold_pred[:, 1])[1] pvalues = (pvalue_val, pvalue_ar) RMSE, pcorr, error_per_song, mean_per_song = evaluate( all_fold_y_test, all_fold_pred, 0) # print( # 'sklearn --> valence: %.4f, arousal: %.4f\n' # 'Pearson Corr --> valence: %.4f, arousal: %.4f \n' # # % (RMSE[0], -1. , pcorr[0][0], -1) # % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0]) # ) s = ( 'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar)) print s log_f.write(s) log_f.close() return RMSE, pcorr, pvalues
def train_lstm( dim_proj=260, # word embeding dimension and LSTM number of hidden units. patience=10, # Number of epoch to wait before early stop if no progress max_epochs=5000, # The maximum number of epoch to run dispFreq=10, # Display to stdout the training progress every N updates decay_c=0., # Weight decay for the classifier applied to the U weights. lrate=0.0001, # Learning rate for sgd (not used for adadelta and rmsprop) n_words=10000, # Vocabulary size optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate). encoder='lstm', # TODO: can be removed must be lstm. saveto='lstm_model.npz', # The best model will be saved there validFreq=370, # Compute the validation error after this number of update. saveFreq=1110, # Save the parameters after every saveFreq updates maxlen=100, # Sequence longer then this get ignored batch_size=16, # The batch size during training. valid_batch_size=64, # The batch size used for validation/test set. fold_id=1, n_ex=None, dim_output=0, # Parameter for extra option noise_std=0., use_dropout=True, # if False slightly faster, but worst test error # This frequently need a bigger model. reload_model=None, # Path to a saved model we want to start from. test_size=-1, # If >0, we keep only this number of test example. ): # Model options model_options = locals().copy() print "LSTM EMO --> model options", model_options print 'Loading data' # train, valid, test = load_data(n_words=n_words, valid_portion=0.05, # maxlen=maxlen) best_ep = 0 NUM_OUTPUT = 2 # A CHANGER SI SUR MON MAC DATADIR = '/baie/corpus/emoMusic/train/' # DATADIR = '..' print '... loading FOLD %d' % fold_id fold = pickle.load( open(DATADIR + '/pkl/fold%d_normed.pkl' % (fold_id), "rb")) X_train, y_train, id_train = load_X_from_fold_to_3dtensor( fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor( fold, 'test', NUM_OUTPUT) # train, valid, test = (X_train, y_train) , (X_test, y_test) , (X_test, y_test) if n_ex == None: ## test only on one dimension 0 = Valence 1 = Arousal train, valid, test = (X_train, y_train[:, :, dim_output]), ( X_test, y_test[:, :, dim_output]), (X_test, y_test[:, :, dim_output]) else: ## test only 30 values ## test only on one dimension train, valid, test = (X_train[0:n_ex, :, :], y_train[0:n_ex, :, 0]), (X_test[:, :, :], y_test[:, :, 0]), (X_test[:, :, :], y_test[:, :, 0]) ## test both dimension # train, valid, test = (X_train[0:n_ex,:,:], y_train[0:n_ex,:,:]) , (X_test[:,:,:], y_test[:,:,:]) , (X_test[:,:,:], y_test[:,:,:]) ##################### A VOIR ############################################# # if test_size > 0: # # The test set is sorted by size, but we want to keep random # # size example. So we must select a random selection of the # # examples. # idx = numpy.arange(len(test[0])) # numpy.random.shuffle(idx) # idx = idx[:test_size] # test = ([test[0][n] for n in idx], [test[1][n] for n in idx]) ########################################################################### ##################### NO YDIM SINCE LINEAR REGRESSION ######################## # ydim = numpy.max(train[1]) + 1 # # model_options['ydim'] = ydim print 'Building model' # This create the initial parameters as numpy ndarrays. # Dict name (string) -> numpy ndarray params = init_params(model_options) if reload_model: load_params('lstm_model.npz', params) # This create Theano Shared Variable from the parameters. # Dict name (string) -> Theano Tensor Shared Variable # params and tparams have different copy of the weights. tparams = init_tparams(params) # use_noise is for dropout # (use_noise, x, mask, # y, f_pred_prob, cost) = build_model(tparams, model_options) (use_noise, x, y, f_pred_prob, cost) = build_model(tparams, model_options) # x0 = tensor.matrix('x0', dtype=config.floatX) # (use_noise, x, mask, # y, f_pred_prob, f_pred, cost) = build_model(x0, tparams, model_options) if decay_c > 0.: decay_c = theano.shared(numpy_floatX(decay_c), name='decay_c') weight_decay = 0. weight_decay += (tparams['U']**2).sum() weight_decay *= decay_c cost += weight_decay # f_cost = theano.function([x, mask, y], cost, name='f_cost') f_cost = theano.function([x, y], cost, name='f_cost') grads = tensor.grad(cost, wrt=tparams.values()) # f_grad = theano.function([x, mask, y], grads, name='f_grad') f_grad = theano.function([x, y], grads, name='f_grad') lr = tensor.scalar(name='lr') # f_grad_shared, f_update = optimizer(lr, tparams, grads, # x, mask, y, cost) f_grad_shared, f_update = optimizer(lr, tparams, grads, x, y, cost) print 'Optimization' # shuffle index for the test and validation sets kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size) kf_test = get_minibatches_idx(len(test[0]), valid_batch_size) print "%d train examples" % len(train[0]) print "%d valid examples" % len(valid[0]) print "%d test examples" % len(test[0]) history_errs = [] tot_cost = [] best_p = None bad_count = 0 if validFreq == -1: validFreq = len(train[0]) / batch_size if saveFreq == -1: saveFreq = len(train[0]) / batch_size uidx = 0 # the number of update done estop = False # early stop start_time = time.time() try: for eidx in xrange(max_epochs): n_samples = 0 # Get new shuffled index for the training set. kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True) for _, train_index in kf: uidx += 1 # use_noise.set_value(1.) use_noise.set_value(0.) # Select the random examples for this minibatch # y = [train[1][t] for t in train_index] # x = [train[0][t]for t in train_index] # Get the data in numpy.ndarray format # This swap the axis! # Return something of shape (minibatch maxlen, n samples) # x, mask, y = prepare_data(x, y) # mask = numpy.ones((dim_proj,1)).astype(theano.config.floatX) # mask = numpy.ones((1,dim_proj)).astype(theano.config.floatX) # x = train[0][train_index].astype('int64') x = train[0][train_index].transpose(1, 0, 2).astype(config.floatX) # x = numpy.array([train[0][t,:,:] for t in train_index] # x = X_train y = train[1][train_index].astype(theano.config.floatX) # y = train[1][train_index,:].astype(theano.config.floatX) # y = y_train[train_index].astype(theano.config.floatX) # cost = f_grad_shared(x, mask, y, allow_input_downcast=True) n_samples += x.shape[1] # print x.shape # cost = f_grad_shared(x, mask, y) # theano.printing.debugprint(f_grad_shared) cost = f_grad_shared(x.transpose(1, 0, 2), y) tot_cost.append(cost) f_update(lrate) if numpy.isnan(cost) or numpy.isinf(cost): print 'NaN detected' return 1., 1., 1. if numpy.mod(uidx, dispFreq) == 0: # print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost print 'Epoch ', eidx, 'Update ', uidx, 'tot_Cost ', numpy.mean( tot_cost) tot_cost = [] if saveto and numpy.mod(uidx, saveFreq) == 0: print 'Saving...', if best_p is not None: params = best_p else: params = unzip(tparams) numpy.savez(saveto, history_errs=history_errs, **params) pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) print 'Done' if numpy.mod(uidx, validFreq) == 0: use_noise.set_value(0.) # train_err = pred_error(f_pred, prepare_data, train, kf) # valid_err = pred_error(f_pred, prepare_data, valid, # kf_valid) # test_err = pred_error(f_pred, prepare_data, test, kf_test) # train_err = cost train_err = pred_loss(f_pred_prob, train, kf) # f_pred_prob([]) valid_err = pred_loss(f_pred_prob, valid, kf_valid) test_err = pred_loss(f_pred_prob, test, kf_test) history_errs.append([valid_err, test_err]) if (uidx == 0 or valid_err <= numpy.array(history_errs)[:, 0].min()): best_p = unzip(tparams) bad_counter = 0 best_ep = eidx # print ('Train ', train_err.flatten()[0], 'Valid ', valid_err, # 'Test ', test_err) print('Train ', train_err, 'Valid ', valid_err, 'Test ', test_err) if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, 0].min()): bad_counter += 1 if bad_counter > patience: print 'Early Stop!' estop = True break print 'Seen %d samples' % n_samples if estop: break except KeyboardInterrupt: print "Training interupted" end_time = time.time() if best_p is not None: zipp(best_p, tparams) else: # best_p = unzip(tparams) use_noise.set_value(0.) kf_train_sorted = get_minibatches_idx(len(train[0]), batch_size) # train_err = pred_error(f_pred, prepare_data, train, kf_train_sorted) # valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) # test_err = pred_error(f_pred, prepare_data, test, kf_test) train_err, tr_corr = pred_loss(f_pred_prob, train, kf_train_sorted, pearsonr=pearsonr) valid_err, val_corr = pred_loss(f_pred_prob, valid, kf_valid, pearsonr=pearsonr) test_err = pred_loss(f_pred_prob, test, kf_test) print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err if saveto: numpy.savez(saveto, train_err=train_err, valid_err=valid_err, test_err=test_err, history_errs=history_errs, **best_p) print 'The code run for %d epochs, with %f sec/epochs' % ( (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))) print >> sys.stderr, ('Training took %.1fs' % (end_time - start_time)) return train_err, valid_err, test_err, tr_corr, val_corr, best_ep
print 'input file: %s' % (nom) fold = pickle.load(open(nom, "rb")) fold_res = dict() for subset in ['train', 'test']: print ' ... creating subset: ', subset # doDuplicates = True # X, y, song_ids, genre_indexes = load_data_from_fold_to_3dtensor_and_genre_info(fold, subset, NUM_OUTPUT, NUM_GENRES) # # print X.shape, y.shape, genre_indexes.shape # # (572, 60, 260) (572, 60, 2) (15, 2) # # instead of (because of duplicates): # # (387, 60, 260) (387, 60, 2) (15, 2) # # # fold_res[subset] = dict() # fold_res[subset]['X'] = X # fold_res[subset]['y'] = y # fold_res[subset]['genre'] = genre_indexes doDuplicates = False X, y, song_ids = load_X_from_fold_to_3dtensor(fold, subset, NUM_OUTPUT) # print X.shape, y.shape, song_ids.shape # (387, 60, 260) (387, 60, 2) () fold_res[subset] = dict() fold_res[subset]['X'] = X fold_res[subset]['y'] = y fold_res[subset]['song_ids'] = song_ids write_fold_to_mat_files(DATADIR, fold_res, fold_id, doNormalize, doDuplicates)