def CDNF(): t = 0 if EQ(True) == True: return True else: u = EQ(True) label .zero t = t + 1 (H[t], S[t], a[t]) = (False, None, u) label .one if EQ(invjoin(H, 'and')) == True: return invjoin(H, 'and') else: u = EQ(invjoin(H, 'and')) I = [] for i in range(0, len(H)): if evaluate(u, H[i])==False: I.append(i) if I==[]: goto .zero mu = [] for i in I: mu.append(walk(u, a[i])) S.append(mu[-1] ^ a[i]) for i in range(0,t): H[i] = MDNF(S[i]) goto .one
def make_move(self, request): """Makes a move. Returns a game state with message. """ game = get_by_urlsafe(request.urlsafe_game_key, Game) #check if the game is over, update the score list if needed if evaluate(game.state): outcome = evaluate(game.state) if outcome == "O": msg = "Win" else: msg = "Lose" game.end_game(msg) game.game_over = True game.put() else: if "-" not in game.state: game.end_game("Draw") game.game_over = True if game.game_over: return game.to_form('Game already over!') #check if it is player's turn if game.player: if request.move not in range(0,9): raise endpoints.BadRequestException('Invalid Move') if game.state[request.move] != "-": raise endpoints.BadRequestException('Invalid Move') #update the board state board = list(game.state) board[request.move] = "O" game.state = ''.join(board) #update movecount game.movecount += 1 #evaluate the result if evaluate(game.state): game.end_game("Win") return game.to_form('You win!') if "-" not in game.state: game.end_game("Draw") return game.to_form('This is a Draw.') msg = "AI's turn" game.history() game.player = False game.put() return game.to_form(msg) else: msg = "This is not your turn" return game.to_form(msg)
def get_random_move(self,request): """Get Random Generated Moves""" game = get_by_urlsafe(request.urlsafe_game_key, Game) #check if the game is over, update the score list if needed if evaluate(game.state): outcome = evaluate(game.state) if outcome == "O": msg = "Win" game.end_game(msg) else: msg = "Lose" game.end_game(msg) game.game_over = True game.put() else: if "-" not in game.state: game.end_game("Draw") game.game_over = True if game.game_over: return game.to_form('Game already over!') #check if it is ai's turn if game.player: msg = "Waiting for your move." return game.to_form(msg) else: #update board with a random move game.state = add_random_move(game.state) #update movecount game.movecount += 1 #evaluate the result if evaluate(game.state): game.end_game("Lose") return game.to_form('You Lose!') if "-" not in game.state: game.end_game("Draw") return game.to_form('This is a Draw.') msg = "Your turn" game.history() game.player = True game.put() return game.to_form(msg)
def evaluate_model(self, k, max_warping_window, train_data, train_label, test_data, test_label): print '--------------------------' print '--------------------------\n' print 'Running for k = ', k print 'Running for w = ', max_warping_window model = KnnDtw(k_neighbours = k, max_warping_window = max_warping_window) model.fit(train_data, train_label) predicted_label, probability = model.predict(test_data) print '\nPredicted : ', predicted_label print 'Actual : ', test_label accuracy, precision, recall, f1score = evaluate(self.labels, predicted_label, test_label) print 'Avg/Total Accuracy :', accuracy print 'Avg/Total Precision :', precision print 'Avg/Total Recall :', recall print 'Avg/Total F1 Score :', f1score
if USE_CUDA and args.model: print("USING CUDA") m = m.cuda() return m if args.use: model = get_model() model.load_state_dict(torch.load(args.use)) for fold, (train, val) in enumerate(train_val_generator): print("FOLD " + str(fold)) train_iter, val_iter, test_iter = data_handler.get_bucket_iterators( (train, val, test), args.batch_size) if args.model == 'CNN' or args.model == 'CNNMulti' or args.model == 'LSTM': p, r, f1, p1, r1, f11 = utils.evaluate(model, val_iter) elif args.model == 'CNNFeatures' or args.model == 'CNNMultiFeatures' or args.model == 'LSTMFeatures': p, r, f1, p1, r1, f11 = utils.evaluate(model, val_iter, has_features=True) p_avg += p r_avg += r f1_avg += f1 p1_avg += p1 r1_avg += r1 f11_avg += f11 if args.model == 'CNN' or args.model == 'CNNMulti' or args.model == 'LSTM': p, r, f1, p1, r1, f11 = utils.evaluate(model, test_iter) elif args.model == "CNNFeatures" or args.model == 'CNNMultiFeatures' or args.model == 'LSTMFeatures': p, r, f1, p1, r1, f11 = utils.evaluate(model,
CM.zero_grad() RM.zero_grad() loss.backward() CM_optim.step() RM_optim.step() if (episode + 1) % 100 == 0: print("episode:", episode + 1, "loss %.4f" % loss.item()) if (episode + 1) % 2000 == 0: # test print("Testing...") gzsl_unseen_accuracy = evaluate(test_features, test_label, np.arange(32), attributes, CM, RM, GPU) gzsl_seen_accuracy = evaluate(test_seen_features, test_seen_label, np.arange(32), attributes, CM, RM, GPU) H = 2 * gzsl_seen_accuracy * gzsl_unseen_accuracy / ( gzsl_unseen_accuracy + gzsl_seen_accuracy) print('seen=%.4f, unseen=%.4f, h=%.4f' % (gzsl_seen_accuracy, gzsl_unseen_accuracy, H)) if H > last_H: # save networks torch.save(CM, "./models/APY_CM.pkl") torch.save(RM, "./models/APY_RM.pkl")
def serve(self): value = self.value.serve() return round(evaluate(self.function, value), 5)
print('error', user_id) elif label_sum == 1: useful_recall.append(g) df_useful_recall = pd.concat(useful_recall, sort=False) log.debug(f'df_useful_recall: {df_useful_recall.head()}') df_useful_recall = df_useful_recall.sort_values( ['user_id', 'sim_score'], ascending=[True, False]).reset_index(drop=True) # 计算相关指标 if mode == 'valid': total = df_query[df_query['click_article_id'] != -1].user_id.nunique() hitrate_5_1, mrr_5_1, hitrate_10_1, mrr_10_1, hitrate_20_1, mrr_20_1, hitrate_40_1, mrr_40_1, hitrate_50_1, mrr_50_1 = evaluate( recall_final[recall_final['label'].notnull()], total) log.debug( f'召回合并后,未删除无用用户指标: {hitrate_5_1}, {mrr_5_1}, {hitrate_10_1}, {mrr_10_1}, {hitrate_20_1}, {mrr_20_1}, {hitrate_40_1}, {mrr_40_1}, {hitrate_50_1}, {mrr_50_1}' ) hitrate_5, mrr_5, hitrate_10, mrr_10, hitrate_20, mrr_20, hitrate_40, mrr_40, hitrate_50, mrr_50 = evaluate( df_useful_recall[df_useful_recall['label'].notnull()], total) log.debug( f'召回合并后,删除无用用户指标: {hitrate_5}, {mrr_5}, {hitrate_10}, {mrr_10}, {hitrate_20}, {mrr_20}, {hitrate_40}, {mrr_40}, {hitrate_50}, {mrr_50}' ) df = df_useful_recall['user_id'].value_counts().reset_index() df.columns = ['user_id', 'cnt'] log.debug(f"删除后平均每个用户召回数量:{df['cnt'].mean()}")
def train(sampling_method, k_folds, data_dir, results_dir, device='cpu', use_international=False, verbose=True): model = 'lgbm' start_time = time.time() if verbose: print("Using device: {}".format(device)) print("Reading train data in...") if use_international: print("Using international class.") X_train, Y_train, feature_labels = get_train(data_dir, one_hot=False, use_international=use_international) categorical_feature = ['age_bucket', 'gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel', 'affiliate_provider', 'first_affiliate_tracked', 'signup_app', 'first_device_type', 'first_browser'] if verbose: print("Successfully loaded data") print("Starting Cross-Validation with {} folds...".format(k_folds)) kf = KFold(n_splits=k_folds) kf.get_n_splits(X_train) params = { 'task': 'train', 'objective': 'multiclass', 'num_class': 12, 'num_leaves': 31, 'lambda_l2': 0.1, 'learning_rate': 0.3, 'feature_fraction': 0.9, 'min_child_weight': 1.0, 'device': device, 'gpu_device_id': 0, 'gpu_platform_id': 0, 'max_bin': 63, 'verbose': 0 } if use_international: params['objective'] = 'binary' del params["num_class"] for k, (train_index, test_index) in enumerate(kf.split(X_train)): print("Processing Fold {} out of {}".format(k+1, k_folds)) X_trainCV, X_testCV = X_train[train_index], X_train[test_index] Y_trainCV, Y_testCV = Y_train[train_index], Y_train[test_index] if verbose: print("{} sampling process started...".format(sampling_method)) curr_time = time.time() if sampling_method == "adasyn": X_train_resampled, Y_train_resampled = ADASYN().fit_sample(X_trainCV, Y_trainCV) elif sampling_method == "smote": X_train_resampled, Y_train_resampled = SMOTE().fit_sample(X_trainCV, Y_trainCV) elif sampling_method == "random": X_train_resampled, Y_train_resampled = RandomOverSampler().fit_sample(X_trainCV, Y_trainCV) elif sampling_method == "smoteenn": X_train_resampled, Y_train_resampled = SMOTEENN().fit_sample(X_trainCV, Y_trainCV) else: X_train_resampled, Y_train_resampled = X_trainCV, Y_trainCV if verbose: print("Oversampling completed") print("Time Taken: {:.2f}".format(time.time()-curr_time)) print("Size of Oversampled data: {}".format(X_train_resampled.shape)) print("{} model(s) selected for classification".format(model)) curr_time = time.time() lgb_train = lgb.Dataset(data=X_train_resampled, label=Y_train_resampled, feature_name=feature_labels, categorical_feature=categorical_feature) clf = lgb.train(params, lgb_train, num_boost_round=30) print("Time taken: {:.2f}".format(time.time()-curr_time)) Y_probs = clf.predict(X_testCV) result = evaluate(Y_testCV, Y_probs) print(result) feature_imp = clf.feature_importance(importance_type='gain') feature_imp = {label: imp for label, imp in zip(feature_labels, feature_imp)} pickle.dump(feature_imp, open(os.path.join(results_dir, "{}_{}_feature_imp_fold_{}.p".format(model, sampling_method, k+1)), "wb" )) pickle.dump(result, open(os.path.join(results_dir, "{}_{}_fold_{}.p".format(model, sampling_method, k+1)), "wb" )) save_examples(X_testCV, Y_testCV, Y_probs, model, sampling_method, k+1, save_dir=results_dir) print("Training took {:.2f}s.".format(time.time()-start_time)) print("Finished.")
comb = itertools.combinations(range(X_train.shape[0]), 2) k = 0 Xp, yp, diff = [], [], [] for (i, j) in comb: if y_train[i] == y_train[j] or blocks_train[i] != blocks_train[j]: continue Xp.append(X_train[i] - X_train[j]) diff.append(y_train[i] - y_train[j]) yp.append(np.sign(diff[-1])) clf = svm.SVC(kernel='linear', C=C, max_iter=100000000) clf.fit(Xp, yp) coef = clf.coef_.ravel() / np.linalg.norm(clf.coef_) rankings = utils.recover_rankings(tasks_test, np.dot(X_test, coef)) score = utils.evaluate(rankings, gold_test) print 'scaler={}, deg={}, C={}: score={}'.format(scaler_name, deg, C, score) if score > best_score: best_score = score best_C = C best_coef = np.copy(coef) best_scaler = scaler best_scaler_name = scaler_name best_deg = deg print '\n\n\ndone' print 'scaler: ', best_scaler_name print 'deg: ', best_deg print 'C: ', best_C print 'coef: ', best_coef
def differential_evolution(crossover=0.9, differential_weight=0.8, population=10, function="rosenbrock", max_iterations=5000, target_fitness=None, dimensions=2): # check crossover in [0,1] # check weight in [0,2] # check pop >= 4 # check either max_iter or target_fitness != None (I get both can be.. Untested for now) # check dimensions > 0 and an Integer """ :param crossover: :param differential_weight: :param pop_size: :return: - init agents x at random positions in search space - while(not end) : (end can be number of iterations, adequate fitness reached, ...) - for each agent : - pick 3 other agents at random - pick a random index (bounded by size of problem) - for each index : - pick a uniformly distributed random number ri ~ U(0,1) - if (ri < crossover) or (i = R ): - yi = ai + F x (bi - ci) - else : - yi = xi - if F(y) <= F(x) : - replace agent x with the improved (or equal) agent y - Pick agent with best fitness and return it as solution """ bounds = [-2.4, 2.4] if function == "rosenbrock" else [-5.12, 5.12] agents = [[ np.random.uniform(low=bounds[0], high=bounds[1]) for _ in range(dimensions) ] for _ in range(population)] for agent in agents: np.clip(agent, bounds[0], bounds[1]) end = False iteration = 0 best_fitness = None agents_history = np.zeros((max_iterations, len(agents), 2), dtype=float) while not end: iteration_history = [] for x in range(len(agents)): new_agent = [0 for _ in range(dimensions)] temp = [] while len(temp) < 3: rand_agent = random.randint(0, len(agents) - 1) if (rand_agent not in temp) and rand_agent != x: temp.append(rand_agent) a = temp[0] b = temp[1] c = temp[2] rand_index = 1 if dimensions < 2 else random.randint(1, dimensions) for index in range(dimensions): ri = np.random.uniform(0.0, 1.0) if (ri < crossover) or (index == rand_index): new_agent[ index] = agents[a][index] + differential_weight * ( agents[b][index] - agents[c][index]) else: new_agent[index] = agents[x][index] if evaluate(function, new_agent) <= evaluate(function, agents[x]): agents[x] = new_agent iteration_history.append(new_agent) agents_history[iteration] = np.clip(iteration_history, bounds[0], bounds[1]) iteration += 1 if (max_iterations != None) and (max_iterations == iteration): end = True if (target_fitness != None) and (target_fitness == best_fitness): end = True # calculate total fitness for all agents fitnesses = np.array([[evaluate(function, agent) for agent in agents] for agents in agents_history]) sum_fitness = [np.sum(fitnesses[:, i]) for i in range(population)] best_agent_id = np.argmin(sum_fitness) best_agent = agents_history[:, best_agent_id, :] best_fitness = fitnesses[:, best_agent_id] return best_agent, best_fitness, agents_history #best_fitness_history
if 0: #Argument prediction FV_arg = feature_vector.FeatureVector('argument') train,valid = utils.create_training_and_validation_file_lists(ratio = 0.75, load=True) Lambda2, misclassification_rates2 = train_perceptron(FV_arg, train, T_max = 20, LR = 10.0, mode='Argument', subs_rate=0.8) plt.plot(misclassification_rates2) (y_hat, y) = test_perceptron(FV_arg, Lambda2, valid, mode='Argument') errors = [1 for y1,y2 in zip(y_hat, y) if y1!=y2] validation_error = len(errors)/float(len(y)) print (validation_error) utils.evaluate(y, y_hat, FV_arg, mode = 'Arguments') savedata2 = (Lambda2,misclassification_rates2) with open('perceptron_argumentbb.data', 'wb') as f: cPickle.dump(savedata2, f) with open('perceptron_argument.data', 'rb') as f: (LLambda2, misc2) = cPickle.load(f) with open('perceptron_argument_predictionsbb.data', 'wb') as f: cPickle.dump((y_hat, y), f) with open('perceptron_argument_predictions.data', 'rb') as f: (yy_hat2, yy2) = cPickle.load(f) if 0: #trigger prediction
all_fold_pred.append(data) all_fold_pred = [item for sublist in all_fold_pred for item in sublist] all_fold_pred = np.array(all_fold_pred, dtype=float) print all_fold_pred.shape, all_fold_y_test.shape # compute t-test p-values with baseline predictions baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl' baseline_preds = pickle.load(open( baseline_prediction_file, 'r' )) pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1] pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1] pvalues = (pvalue_val, pvalue_ar) RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0) # print( # 'sklearn --> valence: %.4f, arousal: %.4f\n' # 'Pearson Corr --> valence: %.4f, arousal: %.4f \n' # # % (RMSE[0], -1. , pcorr[0][0], -1) # % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0]) # ) s = ( 'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar) ) print s
for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) y_hat = np.array(pred, dtype=float) y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' all_fold_pred.append(y_hat.tolist()) all_fold_y_test.append(y_test_concat.tolist()) RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0]) s = ( 'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n' % (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]) ) print s log_f.write(s) doPlot = False if doPlot: fig, ax = plt.subplots() x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0]) if EMO == 'valence': ax.plot(x1, y_test_concat[:, 0], 'o', label="Data") # ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction")
def rnn_main( fold, n_hidden=10, n_epochs=100, lr=0.001, lrd = 0.999, reg_coef= 0.01): doSaveModel = True MODELDIR = 'AE/models/' LOGDIR = MODELDIR if not path.exists(MODELDIR): makedirs(MODELDIR) print '... output dir: %s'%(MODELDIR) # # initialize global logger variable # print '... initializing global logger variable' # logger = logging.getLogger(__name__) # withFile = False # logger = settings.init(MODELDIR + 'train.log', withFile) # perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef) perf_file_name = LOGDIR + 'performance.log' log_f = open(perf_file_name, 'w') all_fold_pred = list() all_fold_y_test = list() all_fold_id_test = list() # for fold_id in range(10): # fold_id = 0 # fold = folds[fold_id] t0 = time.time() # print '... loading FOLD %d'%fold_id # if useEssentia: # fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed_essentia.pkl'%(fold_id), "rb" ) ) X_train = fold['train']['X'] y_train = fold['train']['y'] id_train = fold['train']['song_id'] X_test = fold['test']['X'] y_test = fold['test']['y'] id_test = fold['test']['song_id'] X_train = np.reshape(X_train, (X_train.shape[0]//60, 60, X_train.shape[1]), order='C') X_test = np.reshape(X_test, (X_test.shape[0]//60, 60, X_test.shape[1]), order='C') y_train = np.reshape(y_train, (y_train.shape[0]//60, 60, y_train.shape[1]), order='C') y_test = np.reshape(y_test, (y_test.shape[0]//60, 60, y_test.shape[1]), order='C') # else: # # fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) ) # X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) # X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) print X_train.shape, y_train.shape, X_test.shape, y_test.shape nb_seq_train, nb_frames_train, nb_features_train = X_train.shape nb_seq_test, nb_frames_test, nb_features_test = X_test.shape assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST' assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST' dim_ouput_train = y_train.shape[2] dim_ouput_test = y_test.shape[2] assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST' n_in = nb_features_train n_out = dim_ouput_train n_steps = nb_frames_train validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, learning_rate=lr, learning_rate_decay=lrd, L1_reg=reg_coef, L2_reg=reg_coef, n_epochs=n_epochs, activation='tanh') model.fit(X_train, y_train, validation_frequency=validation_frequency) if doSaveModel: # model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef) model_name = MODELDIR + 'model_fold%d.pkl'%(fold_id) model.save(fpath=model_name) pred = list() for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) y_hat = np.array(pred, dtype=float) y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' # all_fold_pred.append(y_hat.tolist()) # all_fold_y_test.append(y_test_concat.tolist()) RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0]) s = ( 'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n' % (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]) ) print s log_f.write(s) fold_prediction_file = 'AE/pred/fold%d.pkl'%(fold_id) pickle.dump( y_hat, open( fold_prediction_file, "wb" ) ) doPlotTrain = False if doPlotTrain: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_train[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_train[0]) guess = model.predict(X_train[0]) guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() doPlotTest = False if doPlotTest: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_test[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_test[0]) # guess = model.predict(X_test[0]) guess = y_hat[0] guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() print "... Elapsed time: %f" % (time.time() - t0) # all_fold_pred = [item for sublist in all_fold_pred for item in sublist] # all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist] # # all_fold_pred = np.array(all_fold_pred, dtype=float) # all_fold_y_test = np.array(all_fold_y_test, dtype=float) # # print all_fold_pred.shape, all_fold_y_test.shape # # # save predictions # pred_file = LOGDIR + 'all_predictions.pkl' # pickle.dump( all_fold_pred, open( pred_file, "wb" ) ) # print ' ... all predictions saved in: %s'%(pred_file) # # ref_file = 'rnn/all_groundtruth.pkl' # # pickle.dump( all_fold_y_test, open( ref_file, "wb" ) ) # # # compute t-test p-values with baseline predictions # baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl' # baseline_preds = pickle.load(open( baseline_prediction_file, 'r' )) # # pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1] # pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1] # pvalues = (pvalue_val, pvalue_ar) # RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0) # # # print( # # 'sklearn --> valence: %.4f, arousal: %.4f\n' # # 'Pearson Corr --> valence: %.4f, arousal: %.4f \n' # # # % (RMSE[0], -1. , pcorr[0][0], -1) # # % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0]) # # ) # # s = ( # 'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n' # % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar) # ) # # print s # log_f.write(s) log_f.close() # return RMSE, pcorr, pvalues return RMSE
handlers=[ logging.FileHandler("{0}/{1}.log".format('log', args.log_filename)), logging.StreamHandler() ] ) logging.info("Training parameters: {}".format(vars(args))) logging.info("Training for {} epochs.".format(args.nb_epochs)) losses = [] scores = [] scores_tr = [] t1 = time.time() logging.info("**Evaluating initial model...**") with torch.no_grad(): utils.evaluate(model, dl_ev, args.nb_classes) for e in range(1, args.nb_epochs + 1): scheduler.step() time_per_epoch_1 = time.time() losses_per_epoch = [] for x, y in dl_tr: opt.zero_grad() m = model(x.cuda()) loss = criterion(m, y.cuda()) loss.backward() losses_per_epoch.append(loss.data.cpu().numpy()) opt.step() time_per_epoch_2 = time.time() losses.append(np.mean(losses_per_epoch[-20:])) logging.info(
def generate(listing): problem = from_file(["./solomon/" + name])[0] for ind in hall: tmp = evaluate(problem, ind) yield tmp[1], tmp[0]
ScorerInvLength(), ScorerWordNet(), ScorerSWFreqs(), ScorerCorpusComplexity(), ScorerContextSimilarity(), ] def get_features(tasks): features = [] for (sentence, idx), subs in tasks: for sub in subs: features.append(map(lambda scorer: scorer.score((sentence, idx), sub), scorers)) return features print 'initialization done' coefs = [1,1,1,1,1] test_dir = '../data/test-data/' test_tasks = utils.parse_input_file(test_dir) test_gold_file = test_dir + 'substitutions.gold-rankings' test_gold_rankings = utils.parse_rankings_file(test_gold_file) features = get_features(test_tasks) transformer = MinMaxScaler().fit(features) features = transformer.transform(features) rankings = utils.recover_rankings(test_tasks, -np.dot(features, coefs)) print utils.evaluate(rankings, test_gold_rankings) utils.output('unsupervised_rankings', rankings)
def main(): # Argument Settings parser = argparse.ArgumentParser( description='Image Tagging Classification from Naver Shopping Reviews') parser.add_argument('--sess_name', default='example', type=str, help='Session name that is loaded') parser.add_argument('--checkpoint', default='best', type=str, help='Checkpoint') parser.add_argument('--batch_size', default=256, type=int, help='batch size') parser.add_argument('--num_workers', default=16, type=int, help='The number of workers') parser.add_argument('--num_epoch', default=100, type=int, help='The number of epochs') parser.add_argument('--model_name', default='mobilenet_v2', type=str, help='[resnet50, rexnet, dnet1244, dnet1222]') parser.add_argument('--weight_file', default='model.pth', type=str) parser.add_argument('--optimizer', default='SGD', type=str) parser.add_argument('--lr', default=1e-2, type=float) parser.add_argument('--weight_decay', default=1e-5, type=float) parser.add_argument('--learning_anneal', default=1.1, type=float) parser.add_argument('--annealing_period', default=10, type=int) parser.add_argument('--num_gpu', default=1, type=int) parser.add_argument('--pretrain', action='store_true', default=False) parser.add_argument('--mode', default='train', help='Mode') parser.add_argument('--pause', default=0, type=int) parser.add_argument('--iteration', default=0, type=str) args = parser.parse_args() device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # Model logger.info('Build Model') model = select_model(args.model_name, pretrain=args.pretrain, n_class=41) total_param = sum([p.numel() for p in model.parameters()]) logger.info(f'Model size: {total_param} tensors') load_weight(model, args.weight_file) model = model.to(device) nu.bind_model(model) nsml.save('best') if args.pause: nsml.paused(scope=locals()) if args.num_epoch == 0: return # Set the dataset logger.info('Set the dataset') df = pd.read_csv(f'{DATASET_PATH}/train/train_label') train_size = int(len(df) * 0.8) trainset = TagImageDataset(data_frame=df[:train_size], root_dir=f'{DATASET_PATH}/train/train_data', transform=train_transform) testset = TagImageDataset(data_frame=df[train_size:], root_dir=f'{DATASET_PATH}/train/train_data', transform=test_transform) train_loader = DataLoader(dataset=trainset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) test_loader = DataLoader(dataset=testset, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) criterion = nn.CrossEntropyLoss(reduction='mean') optimizer = select_optimizer(model.parameters(), args.optimizer, args.lr, args.weight_decay) criterion = criterion.to(device) if args.mode == 'train': logger.info('Start to train!') train_process(args=args, model=model, train_loader=train_loader, test_loader=test_loader, optimizer=optimizer, criterion=criterion, device=device) elif args.mode == 'test': nsml.load(args.checkpoint, session=args.sess_name) logger.info('[NSML] Model loaded from {}'.format(args.checkpoint)) model.eval() logger.info('Start to test!') test_loss, test_acc, test_f1 = evaluate(model=model, test_loader=test_loader, device=device, criterion=criterion) logger.info(test_loss, test_acc, test_f1)
model.save_mappings(id_to_word, id_to_char, id_to_tag) # Build the model f_train, f_eval = model.build(**parameters) # Reload previous model values if opts.reload: print('Reloading previous model...') model.reload() best_dev = -np.inf best_test = -np.inf dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data, id_to_tag, verbose=verbose) test_score = evaluate(parameters, f_eval, test_sentences, test_data, id_to_tag, verbose=verbose) print("Score on dev: %.5f" % dev_score) print("Score on test: %.5f" % test_score) if dev_score > best_dev: best_dev = dev_score print("New best score on dev.") print("Saving model to disk...") model.save()
model.load_state_dict(torch.load(opts.load+'/model.mdl')) n_epochs = 20 # number of epochs over the training set Division = 2 accuracys = [] precisions = [] recalls = [] FB1s =[] for epoch in xrange(n_epochs): epoch_costs = [] print("Starting epoch %i..." % (epoch)) for i, index in enumerate(np.random.permutation(len(train_data))): if i %(len(train_data)/Division) == 0: # evaluate eval_result = evaluate(model, dev_data, dictionaries) accuracys.append(eval_result['accuracy']) precisions.append(eval_result['precision']) recalls.append(eval_result['recall']) FB1s.append(eval_result['FB1']) save_model_dictionaries('model', model, dictionaries, opts) # Step 1. Remember that Pytorch accumulates gradients. We need to clear them out # before each instance model.zero_grad() # Step 2. Get our inputs ready for the network, that is, turn them into Variables # of word indices. input_words = autograd.Variable(torch.LongTensor(train_data[index]['words'])) targets = autograd.Variable(torch.LongTensor(train_data[index]['tags']))
def train(args): global img, tgt_caption SEED_EVERYTHING() batch_size = args.batch_size epochs = args.epochs device = torch.device(args.device) train_dataloader, valid_dataloader = get_dataloaders(batch_size) with open('config.json','r') as f: model_config = json.load(f) model = get_model(**model_config) #Seq2SeqModel(dropout_p=0.25, hidden_size=256,num_layers=1) model.to(device) # for param in model.encoder.parameters(): # param.requires_grad = False print(model) print(model.decoder.embedding.weight.requires_grad) optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.7, patience=2, verbose=True, min_lr=1e-6, mode='max') if args.resume_from is not None: state = torch.load(args.resume_from) model.load_state_dict(state['model_state_dict']) optimizer.load_state_dict(state['optimizer_state_dict']) scheduler.load_state_dict(state['scheduler_state_dict']) if args.mixed_precision: scaler = torch.cuda.amp.GradScaler() loss_func = nn.CrossEntropyLoss(ignore_index=args.padding_idx) best_bleu = 0 for epoch_i in range(epochs): loss_meter = AverageMeter() bleu_meter = AverageMeter() pbar = tqdm(train_dataloader, total=len(train_dataloader)) model.train() for step, batch in enumerate(pbar): img = batch[0].to(device) tgt_caption = batch[1].to(device) optimizer.zero_grad() if args.mixed_precision: with torch.cuda.amp.autocast(): outputs = model(img, tgt_caption) loss = loss_func(outputs.view(-1, args.padding_idx), tgt_caption[1:].view(-1)) scaler.scale(loss).backward() scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) scaler.step(optimizer) scaler.update() else: outputs = model(img, tgt_caption) loss = loss_func(outputs.view(-1, args.padding_idx), tgt_caption[1:].view(-1)) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() pred_captions = outputs.argmax(2).cpu().numpy() true_captions = batch[1][1:].numpy() bleu = calc_bleu_score(true_captions, pred_captions) loss_meter.update(loss.item()) bleu_meter.update(bleu) pbar.set_postfix({'loss':loss_meter.avg, 'bleu':bleu_meter.avg}) valid_loss, valid_bleu = evaluate(model, valid_dataloader, device, epoch_i, args.key, loss_func) scheduler.step(valid_bleu) if valid_bleu > best_bleu: print('validation bleu improved from %.4f to %.4f'%(best_bleu,valid_loss)) print('saving model...') torch.save({'model_state_dict':model.state_dict(), 'optimizer_state_dict':optimizer.state_dict(), 'scheduler_state_dict':scheduler.state_dict()}, f'saved_models/{args.key}/state.pth') best_bleu = valid_bleu print(f'Epoch: {epoch_i+1}/{epochs}, train loss:{loss_meter.avg:.4f}, train bleu:{bleu_meter.avg:.4f}\nvalid loss: {valid_loss:.4f}, valid bleu: {valid_bleu:.4f}') torch.cuda.empty_cache()
best_test = -np.inf count = 0 for epoch in range(n_epochs): epoch_costs = [] print("Starting epoch %i..." % epoch) go = True for i, index in enumerate(np.random.permutation(len(train_data))): count += 1 input = create_input(train_data[index], parameters, True, singletons) new_cost = f_train(*input) epoch_costs.append(new_cost) if i % 50 == 0 and i > 0 == 0: print("%i, cost average: %f" % (i, np.mean(epoch_costs[-50:]))) if count % freq_eval == 0: go = False dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data, id_to_tag, dico_tags) test_score = evaluate(parameters, f_eval, test_sentences, test_data, id_to_tag, dico_tags) print("Score on dev: %.5f" % dev_score) print("Score on test: %.5f" % test_score) if dev_score > best_dev: best_dev = dev_score print("New best score on dev.") print("Saving model to disk...") model.save() if test_score > best_test: best_test = test_score print("New best score on test.") if go: dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data, id_to_tag, dico_tags)
freq_eval = 1000 # evaluate on dev every freq_eval steps best_dev = -np.inf best_test = -np.inf count = 0 for epoch in xrange(n_epochs): epoch_costs = [] print "Starting epoch %i..." % epoch for i, index in enumerate(np.random.permutation(len(train_data))): count += 1 input = create_input(train_data[index], parameters, True, singletons) new_cost = f_train(*input) epoch_costs.append(new_cost) if i % 50 == 0 and i > 0 == 0: print "%i, cost average: %f" % (i, np.mean(epoch_costs[-50:])) if count % freq_eval == 0: dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data, id_to_tag, dico_tags) test_score = evaluate(parameters, f_eval, test_sentences, test_data, id_to_tag, dico_tags) print "Score on dev: %.5f" % dev_score print "Score on test: %.5f" % test_score if dev_score > best_dev: best_dev = dev_score print "New best score on dev." print "Saving model to disk..." model.save() if test_score > best_test: best_test = test_score print "New best score on test." print "Epoch %i done. Average cost: %f" % (epoch, np.mean(epoch_costs))
def validate(val_loader, net, criterion, optimizer, epoch, train_args, restore, visualize): net.eval() val_loss = AverageMeter() inputs_all, gts_all, predictions_all = [], [], [] for vi, data in enumerate(val_loader): inputs, gts = data N = inputs.size(0) inputs = Variable(inputs, volatile=True).cuda() gts = Variable(gts, volatile=True).cuda() outputs = net(inputs) predictions = outputs.data.max(1)[1].squeeze_(1).squeeze_(0).cpu().numpy() val_loss.update(criterion(outputs, gts).data[0] / N, N) if random.random() > train_args['val_img_sample_rate']: inputs_all.append(None) else: inputs_all.append(inputs.data.squeeze_(0).cpu()) gts_all.append(gts.data.squeeze_(0).cpu().numpy()) predictions_all.append(predictions) acc, acc_cls, mean_iu, fwavacc = evaluate(predictions_all, gts_all, voc.num_classes) if mean_iu > train_args['best_record']['mean_iu']: train_args['best_record']['val_loss'] = val_loss.avg train_args['best_record']['epoch'] = epoch train_args['best_record']['acc'] = acc train_args['best_record']['acc_cls'] = acc_cls train_args['best_record']['mean_iu'] = mean_iu train_args['best_record']['fwavacc'] = fwavacc snapshot_name = 'epoch_%d_loss_%.5f_acc_%.5f_acc-cls_%.5f_mean-iu_%.5f_fwavacc_%.5f_lr_%.10f' % ( epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc, optimizer.param_groups[1]['lr'] ) torch.save(net.state_dict(), os.path.join(ckpt_path, exp_name, snapshot_name + '.pth')) torch.save(optimizer.state_dict(), os.path.join(ckpt_path, exp_name, 'opt_' + snapshot_name + '.pth')) if train_args['val_save_to_img_file']: to_save_dir = os.path.join(ckpt_path, exp_name, str(epoch)) check_mkdir(to_save_dir) val_visual = [] for idx, data in enumerate(zip(inputs_all, gts_all, predictions_all)): if data[0] is None: continue input_pil = restore(data[0]) gt_pil = voc.colorize_mask(data[1]) predictions_pil = voc.colorize_mask(data[2]) if train_args['val_save_to_img_file']: input_pil.save(os.path.join(to_save_dir, '%d_input.png' % idx)) predictions_pil.save(os.path.join(to_save_dir, '%d_prediction.png' % idx)) gt_pil.save(os.path.join(to_save_dir, '%d_gt.png' % idx)) val_visual.extend([visualize(input_pil.convert('RGB')), visualize(gt_pil.convert('RGB')), visualize(predictions_pil.convert('RGB'))]) val_visual = torch.stack(val_visual, 0) val_visual = vutils.make_grid(val_visual, nrow=3, padding=5) writer.add_image(snapshot_name, val_visual) print('--------------------------------------------------------------------') print('[epoch %d], [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f]' % ( epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc)) print('best record: [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f], [epoch %d]' % ( train_args['best_record']['val_loss'], train_args['best_record']['acc'], train_args['best_record']['acc_cls'], train_args['best_record']['mean_iu'], train_args['best_record']['fwavacc'], train_args['best_record']['epoch'])) print('--------------------------------------------------------------------') writer.add_scalar('val_loss', val_loss.avg, epoch) writer.add_scalar('acc', acc, epoch) writer.add_scalar('acc_cls', acc_cls, epoch) writer.add_scalar('mean_iu', mean_iu, epoch) writer.add_scalar('fwavacc', fwavacc, epoch) writer.add_scalar('lr', optimizer.param_groups[1]['lr'], epoch) net.train() return val_loss.avg
def run_latency_changing_topo(train_loader, test_loader, num_rounds, epochs, num_clients, latency_nodes, net='net', optimizer='sgd', lr=0.1): """ Runs a decentralized optimization algorithm for the given learning rate for a number of rounds, over some network. Some nodes send their weights with a one-rounds latency, for the entire execution. The network topology evolves over time. Outputs the accuracies and returns them. Params: train_loader (array): the list of all train datasets, one per client test_loader (array): the list of test datasets, one per client comm_matrix (numpy.array): the communication matric modeling the network num_rounds (int): the number of data exchanges between nodes epochs (int): the number of optimization steps between each communication (minimum 1) num_clients (int): the number of clients in the network latency_nodes (array): the list of delayed nodes net (string): the neural network framework we use optimizer (string): the chosen optimizer, SGD by default lr (double): the learning rate for the optimizaion algorithm Returns: global_model (nn.Module): the final global neural network averaging all the clients client_models (array of Net): the list of all the final client neural networks accs (array): the corresponding accuracies, with the same shape as lrs """ accs = [] global_model, client_models = model_init(num_clients, net) opt = optimizer_init(client_models, lr, optimizer) topos = ['centralized', 'ring', 'grid'] topo = np.random.choice(topos) comm_matrix = create_mixing_matrix(topo, num_clients) loss, test_loss, acc = 0.0, 0.0, 0.0 for r in range(num_rounds): old_client_models = client_models old_topo = topo old_comm_matrix = comm_matrix topo = np.random.choice(topos) # client update for i in range(num_clients): loss += client_update(client_models[i], opt[i], train_loader[i], epoch=epochs) # diffuse params diffuse_params_latency(client_models, comm_matrix, latency_nodes) if (r > 0): diffuse_params_latency( old_client_models, old_comm_matrix, np.setdiff1d(np.array(range(num_clients)), latency_nodes)) print("old topo: {}, new topo: {}".format(old_topo, topo)) average_models(global_model, client_models) test_loss, acc = evaluate(global_model, test_loader) print('%d-th round' % r) print('average train loss %0.3g | test loss %0.3g | test acc: %0.3f' % (loss / num_clients, test_loss, acc)) accs.append(acc) return global_model, client_models, accs
(p_e,g_e, p_a, g_a) = test_perceptron_joint(FV, L_t, L_a, valid, mode = 'Joint_constrained', subsample = False) #save results savedata = (L_t, L_a, misc_t ,misc_a) with open('Joint_perceptron_.data', 'wb') as f: cPickle.dump(savedata, f) with open('Joint_perceptron_.data', 'rb') as f: (L_t2, L_a2, misc_t2 ,misc_a2) = cPickle.load(f) #evaluate trigger predictions errors_e = [1 for y1,y2 in zip(p_e, g_e) if y1!=y2] validation_error_e = len(errors_e)/float(len(g_e)) print 'Trigger prediction error (accuracy)',(validation_error_e) utils.evaluate(g_e, p_e, FV, mode = 'Trigger') #evaluate argument predictions pp_a = [i for sublist in p_a for i in sublist] gg_a = [i for sublist in g_a for i in sublist] errors_a = [1 for y1,y2 in zip(pp_a, gg_a) if y1!=y2] validation_error_a = len(errors_a)/float(len(gg_a)) print 'Argument prediction error (accuracy)',(validation_error_a) utils.evaluate(gg_a, pp_a, FV, mode = 'Arguments') #save predictions with open('Joint_perceptron_predictions_e_.data', 'wb') as f: cPickle.dump((p_e,g_e), f) with open('Joint_perceptron_predictions_a_.data', 'wb') as f: cPickle.dump((p_a,g_a), f) with open('Joint_perceptron_predictions_e_.data', 'rb') as f:
def main(argv=None): # Choose a device and move everything there print("Running on {}".format(device)) parser = argparse.ArgumentParser( description="Train a transformer for a copy task") add_optimizer_arguments(parser) add_transformer_arguments(parser) add_auxiliary_arguments(parser) args = parser.parse_args(argv) print("args:\n-----\n", args) # Make the dataset and the model for model_type in ['rnn', 'lstm', 'transformer']: for max_depth in range(1, 12): ii = 0 while ii < 10: print(max_depth, ii) # skip existing models if os.path.isfile( "/content/drive/My Drive/final_project_material/model_storage/model_" + model_type + "_depth_" + str(max_depth) + "_num_" + str(ii)): ii += 1 continue train_set = CountTaskWithEOS(2 * max_depth + 1, max_depth=max_depth) test_set = CountTaskWithEOS(2 * max_depth + 1, max_depth=max_depth) train_loader = DataLoader(train_set, batch_size=batch_size, pin_memory=device == "cuda") test_loader = DataLoader(test_set, batch_size=batch_size, pin_memory=device == "cuda") if model_type == "transformer": d_model = 8 model = SequencePredictorRecurrentTransformer( d_model=d_model, n_classes=3, sequence_length=128, attention_type="causal-linear", n_layers=1, n_heads=args.n_heads, d_query=d_model, # used to be d_query dropout=args.dropout, softmax_temp=None, attention_dropout=args.attention_dropout, ) else: d_model = 3 model = SequencePredictorRNN(d_model=d_model, n_classes=args.n_classes, n_layers=args.n_layers, dropout=args.dropout, rnn_type=model_type) print(f"Created model:\n{model}") model.to(device) # Start training optimizer = get_optimizer(model.parameters(), args) start_epoch = 1 if args.continue_from: start_epoch = load_model(args.continue_from, model, optimizer, device) lr_schedule = torch.optim.lr_scheduler.LambdaLR( optimizer, lambda e: 1. if e < args.reduce_lr_at else 0.1) for e in range(start_epoch, args.epochs + 1): print('Epoch:', e) print('Training...') train(model, optimizer, train_loader, device) print('Evaluating...') acc = evaluate(model, test_loader, device, return_accuracy=True) lr_schedule.step() if e == 40: break if acc >= 1: save_model( "/content/drive/My Drive/final_project_material/model_storage/model_" + model_type + "_depth_" + str(max_depth) + "_num_" + str(ii), model, optimizer, e) ii += 1 break
def rnn_cv( output_model_dir, model_name, pred_file, data, n_hidden=10, n_epochs=50, lr=0.001, lrd = 0.999, reg_coef= 0.01, doSmoothing=False): doSaveModel = True MODELDIR = output_model_dir LOGDIR = MODELDIR print '... model output dir: %s'%(MODELDIR) # smooth prediction params taille = 12 wts = np.ones(taille-1)*1./taille wts = np.hstack((np.array([1./(2*taille)]), wts, np.array([1./(2*taille)]))) delay = (wts.shape[0]-1) / 2 print '... doSmoothing: %s'%(doSmoothing) # # initialize global logger variable # print '... initializing global logger variable' # logger = logging.getLogger(__name__) # withFile = False # logger = settings.init(MODELDIR + 'train.log', withFile) # perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef) perf_file_name = LOGDIR + 'performance.log' log_f = open(perf_file_name, 'w') all_fold_pred = list() all_fold_y_test = list() all_fold_pred_smooth = list() t0 = time.time() X_train = data['train']['X'] y_train = data['train']['y'] id_train = data['train']['song_id'] X_test = X_train y_test = y_train id_test = id_train print '... training and testing on the same data ...' print X_train.shape, y_train.shape, X_test.shape, y_test.shape nb_seq_train, nb_frames_train, nb_features_train = X_train.shape nb_seq_test, nb_frames_test, nb_features_test = X_test.shape assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST' assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST' dim_ouput_train = y_train.shape[2] dim_ouput_test = y_test.shape[2] assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST' n_in = nb_features_train n_out = dim_ouput_train n_steps = nb_frames_train validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, learning_rate=lr, learning_rate_decay=lrd, L1_reg=reg_coef, L2_reg=reg_coef, n_epochs=n_epochs, activation='tanh') model.fit(X_train, y_train, validation_frequency=validation_frequency) if doSaveModel: # model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef) # model_name = MODELDIR + 'model_baseline_predictions_as_features_431songs_normed.pkl' model_name = MODELDIR + model_name model.save(fpath=model_name) pred = list() pred_smooth = list() for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) if doSmoothing: for ind_seq_test in xrange(nb_seq_test): y_hat = np.array(model.predict(X_test[ind_seq_test]), dtype=float) y_hat_smooth = np.zeros_like(y_hat, dtype=float) y_hat_smooth[:, 0] = np.convolve(y_hat[:, 0], wts, mode='same') y_hat_smooth[:delay, 0] = y_hat[:delay, 0] y_hat_smooth[-delay:, 0] = y_hat[-delay:, 0] y_hat_smooth[:, 1] = np.convolve(y_hat[:, 1], wts, mode='same') y_hat_smooth[:delay, 1] = y_hat[:delay, 1] y_hat_smooth[-delay:, 1] = y_hat[-delay:, 1] pred_smooth.append(y_hat_smooth) y_hat = np.array(pred, dtype=float) y_hat_smooth = np.array(pred_smooth, dtype=float) # save predictions as 3d tensors # pred_file = LOGDIR + 'predictions_train_set_baseline_predictions_as_features_431songs_normed.pkl' if doSmoothing: pickle.dump( y_hat_smooth, open( pred_file, "wb" ) ) else: pickle.dump( y_hat, open( pred_file, "wb" ) ) print ' ... predictions saved in: %s'%(pred_file) y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' all_fold_pred.append(y_hat.tolist()) all_fold_y_test.append(y_test_concat.tolist()) RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0]) s = ( 'training_data valence: %.4f %.4f arousal: %.4f %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]) ) print s log_f.write(s) if doSmoothing: y_hat_smooth = np.reshape(y_hat_smooth, (y_hat_smooth.shape[0]*y_hat_smooth.shape[1], y_hat_smooth.shape[2])) # all_fold_pred_smooth.append(y_hat_smooth.tolist()) RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat_smooth, id_test.shape[0]) s = ( 'training_data valence: %.4f %.4f arousal: %.4f %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]) ) print s log_f.write(s) EMO = 'valence' doPlot = False if doPlot: fig, ax = plt.subplots() x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0]) if EMO == 'valence': ax.plot(x1, y_test_concat[:, 0], 'o', label="Data") # ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction") ax.plot(x1, y_hat[:,0], 'ro', label="OLS prediction") else: ax.plot(x1, y_test_concat[:, 1], 'o', label="Data") ax.plot(x1, y_hat[:,1], 'ro', label="OLS prediction") plt.title(EMO + ' on Train subset') ax.legend(loc="best") plt.show() # plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png') doPlotTrain = False if doPlotTrain: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_train[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_train[0]) guess = model.predict(X_train[0]) guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() doPlotTest = False if doPlotTest: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_test[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_test[0]) # guess = model.predict(X_test[0]) guess = y_hat[0] guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() print "... Elapsed time: %f" % (time.time() - t0) all_fold_pred = [item for sublist in all_fold_pred for item in sublist] all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist] all_fold_pred = np.array(all_fold_pred, dtype=float) all_fold_y_test = np.array(all_fold_y_test, dtype=float) print all_fold_pred.shape, all_fold_y_test.shape return RMSE, pcorr
with open('models/binary/' + model_name + '_config.json', 'w') as f: f.write(binary_model.config.to_json_string()) if do_evaluate: print("loading model...") config = BertConfig('models/binary/' + model_name + '_config.json') eval_model = BertForSequenceClassification(config, num_labels = num_labels) eval_model.load_state_dict(torch.load("models/binary/" + model_name + ".pt")) eval_model.cuda() eval_model.eval() print("loading data...") # Note these are different, see indices pos_dataloader = processor.get_dataloader(DATA_DIR, "neg_dev_matched", tokenizer, batch_size = 1, a_idx = 6, b_idx = 7) neg_dataloader = processor.get_dataloader(DATA_DIR, "neg_dev_matched", tokenizer, batch_size = 1, a_idx = 8, b_idx = 7) evaluate(eval_model, pos_dataloader, neg_dataloader, "experiments/may12_3/binary_finetune", 3, DEBUG=True) if other: config = BertConfig('models/binary/' + model_name + '_config.json') eval_model = BertForSequenceClassification(config, num_labels = num_labels) eval_model.load_state_dict(torch.load("models/binary/" + model_name + ".pt")) eval_model.to('cuda:0') eval_model.eval() eval_pos_dataloader = processor.get_dataloader(DATA_DIR, "binary_dev_mismatched", tokenizer, batch_size = 10, a_idx = 6, b_idx = 7, label_idx = 5) #eval_neg_dataloader = processor.get_dataloader(DATA_DIR, "neg_binary_dev_mismatched", tokenizer, batch_size = 10, a_idx = 8, b_idx = 7, label_idx = 5) simple_evaluate(eval_model, eval_pos_dataloader, "experiments/binary_" + model_name + "_pos_preds") #simple_evaluate(eval_model, eval_neg_dataloader, "experiments/binary_finetune_neg_preds")
# here is your result file which will be evaluated result_file = "../data/result.txt" # test set you separate from dataset test_file = "../data/test.txt" if __name__ == "__main__": if len(sys.argv) == 3: result_file = sys.argv[1] test_file = sys.argv[2] elif len(sys.argv) != 1: print("Usage: %s <result file> <test file>\n" % sys.argv[0]) exit(0) result_graph = Graph(result_file) test_graph = Graph(test_file) result_graph.read() test_graph.read() result_set = set() test_set = set() result_edge = result_graph.all_edges() for a, link in result_edge.items(): for b in link: result_set.add((a, b)) test_edge = test_graph.all_edges() for a, link in test_edge.items(): for b in link: test_set.add((a, b)) evaluate(test_set, result_set)
# for i, data in enumerate(train_data): # input = create_input(data, parameters, True, use_gaze, pos, singletons) # train_cost.append(f_plot_cost(*input)) # dev_cost = [] # for i, data in enumerate(dev_data): # input = create_input(data, parameters, True, use_gaze, pos, singletons) # dev_cost.append(f_plot_cost(*input)) # test_cost = [] # for i, data in enumerate(test_data): # input = create_input(data, parameters, True, use_gaze, pos, singletons) # test_cost.append(f_plot_cost(*input)) print "dev:" dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data, id_to_tag, tag_to_id, use_gaze, pos, post_process, list_prefix, label_cnt) print "test:" test_score = evaluate(parameters, f_eval, test_sentences, test_data, id_to_tag, tag_to_id, use_gaze, pos, post_process, list_prefix, label_cnt) fw.write('\t\t'.join([str(epoch), str(dev_score), str(test_score)]) + '\n') # cost_str = '\t\t'.join( # [str(epoch), str(np.mean(train_cost)), str(np.mean(dev_cost)), str(np.mean(test_cost)), str(dev_score), str(test_score)]) # fw.write(cost_str + '\n') #print "Score on train: %.5f" % train_score print "Score on dev: %.5f" % dev_score print "Score on test: %.5f" % test_score if dev_score > best_dev: best_dev = dev_score print "New best score on dev."
model, params = models.create_model(m, n_inputs, n_outputs) if torch.cuda.device_count() > 0: print("Let's use", torch.cuda.device_count(), "GPUs!") model = nn.DataParallel(model) model.cuda() model = utils.load_net_weights(model, best_model) model.eval() dataset_loader = torch.utils.data.DataLoader( dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.n_threads, worker_init_fn=lambda x: utils.worker_init_fn(x)) temp_results = utils.evaluate(dataset, dataset_loader, model, save_prediction=args.save_prediction, model_name=model_info['name']) results = pd.concat([results, temp_results]) #------------------------------------------------------------------------------- # Save evaluation print(results.head()) try: os.mkdir('./metrics/') except: pass if args.name == '': name = './metrics/{}_{}_{}_{}x{}x{}_{}.csv'.format( model_info['name'], args.batch_size, args.n_subvolumes, args.sv_d, args.sv_h, args.sv_w, args.seed) else:
Division = 2 accuracys = [] precisions = [] recalls = [] FB1s = [] for epoch in xrange( n_epochs ): # again, normally you would NOT do 300 epochs, it is toy data epoch_costs = [] print("Starting epoch %i..." % (epoch)) for i, index in enumerate(np.random.permutation(len(train_data))): if i % (len(train_data) / Division) == 0: # evaluate eval_result = evaluate(model, dev_data, dictionaries, opts.lower) accuracys.append(eval_result['accuracy']) precisions.append(eval_result['precision']) recalls.append(eval_result['recall']) FB1s.append(eval_result['FB1']) # Step 1. Remember that Pytorch accumulates gradients. We need to clear them out # before each instance model.zero_grad() # Step 2. Get our inputs ready for the network, that is, turn them into Variables # of word indices. input_words = autograd.Variable( torch.LongTensor(train_data[index]['words'])) targets = autograd.Variable(torch.LongTensor( train_data[index]['tags']))
best_dev = -np.inf best_test = -np.inf count = 0 for epoch in xrange(n_epochs): epoch_costs = [] print "Starting epoch %i..." % epoch for i, index in enumerate(np.random.permutation(len(train_data))): count += 1 input = create_input(train_data[index], parameters, True, singletons) new_cost = f_train(*input) epoch_costs.append(new_cost) if i % 50 == 0 and i > 0 == 0: print "%i, cost average: %f" % (i, np.mean(epoch_costs[-50:])) if count % freq_eval == 0: dev_score, dev_eval_lines_prob = evaluate(parameters, f_eval, dev_sentences, dev_data, id_to_tag, dico_tags) test_score, test_eval_lines_prob = evaluate( parameters, f_eval, test_sentences, test_data, id_to_tag, dico_tags) print "Score on dev: %.5f" % dev_score print "Score on test: %.5f" % test_score if dev_score > best_dev: best_dev = dev_score best_dev_eval_lines = dev_eval_lines_prob test_with_best_dev = test_score print "New best score on dev." print "Saving model to disk..." model.save() if test_score > best_test: best_test = test_score
def rnn_cv(output_model_dir, model_name, pred_file, data, n_hidden=10, n_epochs=50, lr=0.001, lrd=0.999, reg_coef=0.01, doSmoothing=False): doSaveModel = True MODELDIR = output_model_dir LOGDIR = MODELDIR print '... model output dir: %s' % (MODELDIR) # smooth prediction params taille = 12 wts = np.ones(taille - 1) * 1. / taille wts = np.hstack( (np.array([1. / (2 * taille)]), wts, np.array([1. / (2 * taille)]))) delay = (wts.shape[0] - 1) / 2 print '... doSmoothing: %s' % (doSmoothing) # # initialize global logger variable # print '... initializing global logger variable' # logger = logging.getLogger(__name__) # withFile = False # logger = settings.init(MODELDIR + 'train.log', withFile) # perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef) perf_file_name = LOGDIR + 'performance.log' log_f = open(perf_file_name, 'w') all_fold_pred = list() all_fold_y_test = list() all_fold_pred_smooth = list() t0 = time.time() X_train = data['train']['X'] y_train = data['train']['y'] id_train = data['train']['song_id'] X_test = X_train y_test = y_train id_test = id_train print '... training and testing on the same data ...' print X_train.shape, y_train.shape, X_test.shape, y_test.shape nb_seq_train, nb_frames_train, nb_features_train = X_train.shape nb_seq_test, nb_frames_test, nb_features_test = X_test.shape assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST' assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST' dim_ouput_train = y_train.shape[2] dim_ouput_test = y_test.shape[2] assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST' n_in = nb_features_train n_out = dim_ouput_train n_steps = nb_frames_train validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, learning_rate=lr, learning_rate_decay=lrd, L1_reg=reg_coef, L2_reg=reg_coef, n_epochs=n_epochs, activation='tanh') model.fit(X_train, y_train, validation_frequency=validation_frequency) if doSaveModel: # model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef) # model_name = MODELDIR + 'model_baseline_predictions_as_features_431songs_normed.pkl' model_name = MODELDIR + model_name model.save(fpath=model_name) pred = list() pred_smooth = list() for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) if doSmoothing: for ind_seq_test in xrange(nb_seq_test): y_hat = np.array(model.predict(X_test[ind_seq_test]), dtype=float) y_hat_smooth = np.zeros_like(y_hat, dtype=float) y_hat_smooth[:, 0] = np.convolve(y_hat[:, 0], wts, mode='same') y_hat_smooth[:delay, 0] = y_hat[:delay, 0] y_hat_smooth[-delay:, 0] = y_hat[-delay:, 0] y_hat_smooth[:, 1] = np.convolve(y_hat[:, 1], wts, mode='same') y_hat_smooth[:delay, 1] = y_hat[:delay, 1] y_hat_smooth[-delay:, 1] = y_hat[-delay:, 1] pred_smooth.append(y_hat_smooth) y_hat = np.array(pred, dtype=float) y_hat_smooth = np.array(pred_smooth, dtype=float) # save predictions as 3d tensors # pred_file = LOGDIR + 'predictions_train_set_baseline_predictions_as_features_431songs_normed.pkl' if doSmoothing: pickle.dump(y_hat_smooth, open(pred_file, "wb")) else: pickle.dump(y_hat, open(pred_file, "wb")) print ' ... predictions saved in: %s' % (pred_file) y_hat = np.reshape(y_hat, (y_hat.shape[0] * y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape( y_test, (y_test.shape[0] * y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' all_fold_pred.append(y_hat.tolist()) all_fold_y_test.append(y_test_concat.tolist()) RMSE, pcorr, error_per_song, mean_per_song = evaluate( y_test_concat, y_hat, id_test.shape[0]) s = ('training_data valence: %.4f %.4f arousal: %.4f %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])) print s log_f.write(s) if doSmoothing: y_hat_smooth = np.reshape( y_hat_smooth, (y_hat_smooth.shape[0] * y_hat_smooth.shape[1], y_hat_smooth.shape[2])) # all_fold_pred_smooth.append(y_hat_smooth.tolist()) RMSE, pcorr, error_per_song, mean_per_song = evaluate( y_test_concat, y_hat_smooth, id_test.shape[0]) s = ('training_data valence: %.4f %.4f arousal: %.4f %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])) print s log_f.write(s) EMO = 'valence' doPlot = False if doPlot: fig, ax = plt.subplots() x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0]) if EMO == 'valence': ax.plot(x1, y_test_concat[:, 0], 'o', label="Data") # ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction") ax.plot(x1, y_hat[:, 0], 'ro', label="OLS prediction") else: ax.plot(x1, y_test_concat[:, 1], 'o', label="Data") ax.plot(x1, y_hat[:, 1], 'ro', label="OLS prediction") plt.title(EMO + ' on Train subset') ax.legend(loc="best") plt.show() # plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png') doPlotTrain = False if doPlotTrain: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_train[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_train[0]) guess = model.predict(X_train[0]) guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() doPlotTest = False if doPlotTest: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_test[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_test[0]) # guess = model.predict(X_test[0]) guess = y_hat[0] guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() print "... Elapsed time: %f" % (time.time() - t0) all_fold_pred = [item for sublist in all_fold_pred for item in sublist] all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist] all_fold_pred = np.array(all_fold_pred, dtype=float) all_fold_y_test = np.array(all_fold_y_test, dtype=float) print all_fold_pred.shape, all_fold_y_test.shape return RMSE, pcorr
input_zip = ZipFile(input_zip) return {name: input_zip.read(name) for name in input_zip.namelist()} data = extract_zip('/media/dsshim/nyu_v2/nyu_test.zip') from io import BytesIO rgb = np.load(BytesIO(data['eigen_test_rgb.npy'])) depth = np.load(BytesIO(data['eigen_test_depth.npy'])) crop = np.load(BytesIO(data['eigen_test_crop.npy'])) print('Test data loaded.\n') if args.model_type == 'densedepth': model = Model() else: model = ResNet(layers=args.layers) model.load_state_dict( torch.load('checkpoints/%s_%d.pth' % (args.model_type, args.layers))) model = model.cuda() model.eval() start = time.time() print('Testing...') e = evaluate(model, rgb, depth, crop, batch_size=1) end = time.time() print('\nTest time', end - start, 's')
def run(om,options,i): if options.multiling: outdir = options.outdir else: cur_treebank = om.languages[i] outdir = cur_treebank.outdir if options.shared_task: outdir = options.shared_task_outdir if not options.predict: # training fineTune = False start_from = 1 if options.continueModel is None: continueTraining = False else: continueTraining = True trainedModel = options.continueModel if options.fineTune: fineTune = True else: start_from = options.first_epoch - 1 if not continueTraining: print 'Preparing vocab' if options.multiling: path_is_dir=True, words, w2i, pos, cpos, rels, langs, ch = utils.vocab(om.languages,\ path_is_dir, options.shareWordLookup,\ options.shareCharLookup) else: words, w2i, pos, cpos, rels, langs, ch = utils.vocab(cur_treebank.trainfile) paramsfile = os.path.join(outdir, options.params) with open(paramsfile, 'w') as paramsfp: print 'Saving params to ' + paramsfile pickle.dump((words, w2i, pos, rels, cpos, langs, options, ch), paramsfp) print 'Finished collecting vocab' else: paramsfile = os.path.join(outdir, options.params) with open(paramsfile, 'rb') as paramsfp: print 'Load params from ' + paramsfile words, w2i, pos, rels, cpos, langs, options, ch = pickle.load(paramsfp) print 'Finished loading vocab' max_epochs = options.first_epoch + options.epochs print 'Initializing blstm arc hybrid:' parser = ArcHybridLSTM(words, pos, rels, cpos, langs, w2i, ch, options) if continueTraining: if not fineTune: # continue training only, not doing fine tuning options.first_epoch = start_from + 1 max_epochs = options.epochs else: # fine tune model options.first_epoch = options.epochs + 1 max_epochs = options.first_epoch + 15 print 'Fine tune model for another', max_epochs - options.first_epoch, 'epochs' parser.Load(trainedModel) best_multi_las = -1 best_multi_epoch = 0 if continueTraining: train_stats = codecs.open(os.path.join(outdir, 'train.stats'), 'a', encoding='utf-8') else: train_stats = codecs.open(os.path.join(outdir, 'train.stats'), 'w', encoding='utf-8') for epoch in xrange(options.first_epoch, max_epochs + 1): print 'Starting epoch ' + str(epoch) if options.multiling: traindata = list(utils.read_conll_dir(om.languages, "train", options.max_sentences)) else: traindata = list(utils.read_conll(cur_treebank.trainfile, cur_treebank.iso_id,options.max_sentences)) parser.Train(traindata) train_stats.write(unicode('Epoch ' + str(epoch) + '\n')) print 'Finished epoch ' + str(epoch) model_file = os.path.join(outdir, options.model + '.tmp') parser.Save(model_file) if options.pred_dev: # use the model to predict on dev data if options.multiling: pred_langs = [lang for lang in om.languages if lang.pred_dev] # languages which have dev data on which to predict for lang in pred_langs: lang.outfilename = os.path.join(lang.outdir, 'dev_epoch_' + str(epoch) + '.conllu') print "Predicting on dev data for " + lang.name devdata = utils.read_conll_dir(pred_langs,"dev") pred = list(parser.Predict(devdata)) if len(pred)>0: utils.write_conll_multiling(pred,pred_langs) else: print "Warning: prediction empty" if options.pred_eval: total_las = 0 for lang in pred_langs: print "Evaluating dev prediction for " + lang.name las_score = utils.evaluate(lang.dev_gold, lang.outfilename,om.conllu) total_las += las_score train_stats.write(unicode('Dev LAS ' + lang.name + ': ' + str(las_score) + '\n')) if options.model_selection: if total_las > best_multi_las: best_multi_las = total_las best_multi_epoch = epoch else: # monolingual case if cur_treebank.pred_dev: print "Predicting on dev data for " + cur_treebank.name devdata = utils.read_conll(cur_treebank.devfile, cur_treebank.iso_id) cur_treebank.outfilename = os.path.join(outdir, 'dev_epoch_' + str(epoch) + ('.conll' if not om.conllu else '.conllu')) pred = list(parser.Predict(devdata)) utils.write_conll(cur_treebank.outfilename, pred) if options.pred_eval: print "Evaluating dev prediction for " + cur_treebank.name las_score = utils.evaluate(cur_treebank.dev_gold, cur_treebank.outfilename, om.conllu) if options.model_selection: if las_score > cur_treebank.dev_best[1]: cur_treebank.dev_best = [epoch, las_score] train_stats.write(unicode('Dev LAS ' + cur_treebank.name + ': ' + str(las_score) + '\n')) if epoch == max_epochs: # at the last epoch choose which model to copy to barchybrid.model if not options.model_selection: best_epoch = options.epochs # take the final epoch if model selection off completely (for example multilingual case) else: if options.multiling: best_epoch = best_multi_epoch else: best_epoch = cur_treebank.dev_best[0] # will be final epoch by default if model selection not on for this treebank if cur_treebank.model_selection: print "Best dev score of " + str(cur_treebank.dev_best[1]) + " found at epoch " + str(cur_treebank.dev_best[0]) bestmodel_file = os.path.join(outdir,"barchybrid.model.tmp") model_file = os.path.join(outdir,"barchybrid.model") if fineTune: model_file = os.path.join(outdir,"barchybrid.tuned.model") print "Best epoch: " + str(best_epoch) print "Copying " + bestmodel_file + " to " + model_file copyfile(bestmodel_file,model_file) train_stats.close() else: #if predict - so # import pdb;pdb.set_trace() eval_type = options.evaltype print "Eval type: ", eval_type if eval_type == "train": if options.multiling: for l in om.languages: l.test_gold = l.test_gold.replace('test', 'train') else: cur_treebank.testfile = cur_treebank.trainfile cur_treebank.test_gold = cur_treebank.trainfile elif eval_type == "dev": if options.multiling: for l in om.languages: l.test_gold = l.test_gold.replace('test', 'dev') else: cur_treebank.testfile = cur_treebank.devfile cur_treebank.test_gold = cur_treebank.devfile if options.multiling: modeldir = options.modeldir if options.fineTune: prefix = [os.path.join(outdir, os.path.basename(l.test_gold) + '-tuned') for l in om.languages] else: prefix = [os.path.join(outdir, os.path.basename(l.test_gold)) for l in om.languages] else: modeldir = om.languages[i].modeldir if options.fineTune: prefix = os.path.join(outdir, os.path.basename(cur_treebank.testfile)) + '-tuned' else: prefix = os.path.join(outdir, os.path.basename(cur_treebank.testfile)) if not options.extract_vectors: prefix = None params = os.path.join(modeldir, options.params) print 'Reading params from ' + params with open(params, 'r') as paramsfp: words, w2i, pos, rels, cpos, langs, stored_opt, ch = pickle.load(paramsfp) parser = ArcHybridLSTM(words, pos, rels, cpos, langs, w2i, ch, stored_opt) if options.fineTune: options.model = options.model.replace('.model', '.tuned.model') model = os.path.join(modeldir, options.model) parser.Load(model) if options.multiling: testdata = utils.read_conll_dir(om.languages, eval_type) else: testdata = utils.read_conll(cur_treebank.testfile, cur_treebank.iso_id) ts = time.time() if options.multiling: for l in om.languages: l.outfilename = os.path.join(outdir, eval_type + "-" + l.outfilename) pred = list(parser.Predict(testdata, prefix)) utils.write_conll_multiling(pred,om.languages) else: if cur_treebank.outfilename: cur_treebank.outfilename = os.path.join(outdir, eval_type + "-" + cur_treebank.outfilename) else: cur_treebank.outfilename = os.path.join(outdir, 'out' + ('.conll' if not om.conllu else '.conllu')) utils.write_conll(cur_treebank.outfilename, parser.Predict(testdata, prefix)) te = time.time() if options.pred_eval: if options.multiling: for l in om.languages: print "Evaluating on " + l.name score = utils.evaluate(l.test_gold, l.outfilename, om.conllu) print "Obtained LAS F1 score of %.2f on %s" %(score, l.name) else: print "Evaluating on " + cur_treebank.name score = utils.evaluate(cur_treebank.test_gold, cur_treebank.outfilename, om.conllu) print "Obtained LAS F1 score of %.2f on %s" %(score,cur_treebank.name) print 'Finished predicting'
raise AssertionError("Wrong Model Type!") criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) criterion.to(device) best_val_acc = 0.0 train_losses, train_accuracies = [], [] valid_losses, valid_accuracies = [], [] for epoch in range(NUM_EPOCHS): train_loss, train_accuracy = train(model, device, train_loader, criterion, optimizer, epoch) valid_loss, valid_accuracy, valid_results = evaluate(model, device, valid_loader, criterion) train_losses.append(train_loss) valid_losses.append(valid_loss) train_accuracies.append(train_accuracy) valid_accuracies.append(valid_accuracy) is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric. if is_best: best_val_acc = valid_accuracy torch.save(model, os.path.join(PATH_OUTPUT, save_file)) plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies) best_model = torch.load(os.path.join(PATH_OUTPUT, save_file))
def main(): cfg = TrainConfig().parse() print(cfg.name) result_dir = os.path.join( cfg.result_root, cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')) if not os.path.isdir(result_dir): os.makedirs(result_dir) utils.write_configure_to_file(cfg, result_dir) np.random.seed(seed=cfg.seed) # prepare dataset train_session = cfg.train_session train_set = prepare_dataset(cfg.feature_root, train_session, cfg.feat, cfg.label_root) batch_per_epoch = len(train_set) // cfg.sess_per_batch val_session = cfg.val_session val_set = prepare_dataset(cfg.feature_root, val_session, cfg.feat, cfg.label_root) # construct the graph with tf.Graph().as_default(): tf.set_random_seed(cfg.seed) global_step = tf.Variable(0, trainable=False) lr_ph = tf.placeholder(tf.float32, name='learning_rate') # load backbone model if cfg.network == "tsn": model = networks.ConvTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) elif cfg.network == "rtsn": model = networks.ConvRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim) # get the embedding input_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, None, None, None]) label_ph = tf.placeholder(tf.float32, shape=[None]) dropout_ph = tf.placeholder(tf.float32, shape=[]) model.forward(input_ph, dropout_ph) if cfg.normalized: embedding = tf.nn.l2_normalize(model.hidden, axis=-1, epsilon=1e-10) else: embedding = model.hidden # variable for visualizing the embeddings emb_var = tf.Variable([0.0], name='embeddings') set_emb = tf.assign(emb_var, embedding, validate_shape=False) # calculated for monitoring all-pair embedding distance diffs = utils.all_diffs_tf(embedding, embedding) all_dist = utils.cdist_tf(diffs) tf.summary.histogram('embedding_dists', all_dist) # whether to use softplus if cfg.no_soft: margin = cfg.alpha else: margin = "soft" metric_loss, num_active, diff, weights, fp, cn = networks.batch_hard( all_dist, label_ph, margin) regularization_loss = tf.reduce_sum( tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) total_loss = metric_loss + regularization_loss * cfg.lambda_l2 tf.summary.scalar('active_ratio', num_active) tf.summary.scalar('learning_rate', lr_ph) train_op = utils.optimize(total_loss, global_step, cfg.optimizer, lr_ph, tf.global_variables()) saver = tf.train.Saver(max_to_keep=10) summary_op = tf.summary.merge_all() # session iterator for session sampling feat_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) label_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch]) train_data = session_generator(feat_paths_ph, label_paths_ph, sess_per_batch=cfg.sess_per_batch, num_threads=2, shuffled=False, preprocess_func=model.prepare_input) train_sess_iterator = train_data.make_initializable_iterator() next_train = train_sess_iterator.get_next() # prepare validation data val_feats = [] val_labels = [] for session in val_set: eve_batch, lab_batch, _ = load_data_and_label( session[0], session[1], model.prepare_input_test ) # use prepare_input_test for testing time val_feats.append(eve_batch) val_labels.append(lab_batch) val_feats = np.concatenate(val_feats, axis=0) val_labels = np.concatenate(val_labels, axis=0) print("Shape of val_feats: ", val_feats.shape) # generate metadata.tsv for visualize embedding with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout: for v in val_labels: fout.write('%d\n' % int(v)) # Start running the graph if cfg.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu gpu_options = tf.GPUOptions(allow_growth=True) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) summary_writer = tf.summary.FileWriter(result_dir, sess.graph) with sess.as_default(): sess.run(tf.global_variables_initializer()) # load pretrain model, if needed if cfg.model_path: print("Restoring pretrained model: %s" % cfg.model_path) saver.restore(sess, cfg.model_path) ################## Training loop ################## epoch = -1 while epoch < cfg.max_epochs - 1: step = sess.run(global_step, feed_dict=None) epoch = step // batch_per_epoch # learning rate schedule, reference: "In defense of Triplet Loss" if epoch < cfg.static_epochs: learning_rate = cfg.learning_rate else: learning_rate = cfg.learning_rate * \ 0.001**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs)) # prepare data for this epoch random.shuffle(train_set) feat_paths = [path[0] for path in train_set] label_paths = [path[1] for path in train_set] # reshape a list to list of list # interesting hacky code from: https://stackoverflow.com/questions/10124751/convert-a-flat-list-to-list-of-list-in-python feat_paths = list(zip(*[iter(feat_paths)] * cfg.sess_per_batch)) label_paths = list( zip(*[iter(label_paths)] * cfg.sess_per_batch)) sess.run(train_sess_iterator.initializer, feed_dict={ feat_paths_ph: feat_paths, label_paths_ph: label_paths }) # for each epoch batch_count = 1 while True: try: # First, sample sessions for a batch start_time_select = time.time() eve, se, lab = sess.run(next_train) select_time1 = time.time() - start_time_select # Second, select samples for a batch batch_idx = select_batch(lab, cfg.batch_size) eve = eve[batch_idx] lab = lab[batch_idx] # Third, perform training on a batch start_time_train = time.time() err, _, step, summ, diff_v, weights_v, fp_v, cn_v, dist_v = sess.run( [ total_loss, train_op, global_step, summary_op, diff, weights, fp, cn, all_dist ], feed_dict={ input_ph: eve, dropout_ph: cfg.keep_prob, label_ph: np.squeeze(lab), lr_ph: learning_rate }) train_time = time.time() - start_time_train pdb.set_trace() print ("Epoch: [%d][%d/%d]\tEvent num: %d\tSelect_time: %.3f\tTrain_time: %.3f\tLoss %.4f" % \ (epoch+1, batch_count, batch_per_epoch, eve.shape[0], select_time1, train_time, err)) summary = tf.Summary(value=[ tf.Summary.Value(tag="train_loss", simple_value=err), tf.Summary.Value(tag="select_time1", simple_value=select_time1) ]) summary_writer.add_summary(summary, step) summary_writer.add_summary(summ, step) batch_count += 1 except tf.errors.OutOfRangeError: print("Epoch %d done!" % (epoch + 1)) break # validation on val_set print("Evaluating on validation set...") val_embeddings, _ = sess.run([embedding, set_emb], feed_dict={ input_ph: val_feats, dropout_ph: 1.0 }) mAP, _ = utils.evaluate(val_embeddings, val_labels) summary = tf.Summary(value=[ tf.Summary.Value(tag="Valiation mAP", simple_value=mAP) ]) summary_writer.add_summary(summary, step) # config for embedding visualization config = projector.ProjectorConfig() visual_embedding = config.embeddings.add() visual_embedding.tensor_name = emb_var.name visual_embedding.metadata_path = os.path.join( result_dir, 'metadata_val.tsv') projector.visualize_embeddings(summary_writer, config) # save model saver.save(sess, os.path.join(result_dir, cfg.name + '.ckpt'), global_step=step)
from utils import evaluate # here is your result file which will be evaluated result_file = "../data/result.txt" # test set you separate from dataset test_file = "../data/test.txt" if __name__ == "__main__": if len(sys.argv) == 3: result_file = sys.argv[1] test_file = sys.argv[2] elif len(sys.argv) != 1: print("Usage: %s <result file> <test file>\n" % sys.argv[0]) exit(0) result_graph = Graph(result_file) test_graph = Graph(test_file) result_graph.read() test_graph.read() result_set = set() test_set = set() result_edge = result_graph.all_edges() for a, link in result_edge.items(): for b in link: result_set.add((a, b)) test_edge = test_graph.all_edges() for a, link in test_edge.items(): for b in link: test_set.add((a, b)) evaluate(test_set, result_set)
# MUTATION if np.random.rand() < mutation: new_agent = mutate(new_agent, bounds[0], bounds[1]) agents[x] = new_agent iteration_history.append(new_agent) agents_history[iteration] = np.clip(iteration_history, bounds[0], bounds[1]) fitness_history[iteration] = [ evaluate(function, agent) for agent in agents ] iteration += 1 if (max_iterations != None) and (max_iterations == iteration): end = True if (target_fitness != None) and (target_fitness == best_fitness): end = True # calculate total fitness for all agents fitnesses = np.array([[evaluate(function, agent) for agent in agents] for agents in agents_history]) best_agent_id = int(np.argmin(fitnesses) / max_iterations) % population best_agent = agents_history[:, best_agent_id, :] best_fitness = fitnesses[:, best_agent_id] return best_agent, best_fitness, agents_history #best_fitness_history if __name__ == "__main__": best_agent, best_fitness, agents_history = differential_evolution() print(evaluate("rosenbrock", best_agent))
tst_song = len(song_id_tst) # Create linear regression object regr = linear_model.LinearRegression() # Train the model using the training sets regr.fit(X_train, y_train) # The coefficients # print('Coefficients: \n', regr.coef_) # What we predict y_hat = regr.predict(X_test) RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test, y_hat, tst_song) All_stack =np.hstack(( error_per_song, mean_per_song )) print' Error per song (ar/val) Mean_per_song (ar/val) :\n' print(All_stack) print '\n' print'song_id :' print(song_id_tst) print '\n' #print('Error per song: \n', Error_per_song) print( 'sklearn --> arrousal : %.4f, valence : %.4f\n' 'Pearson Corr --> arrousal : %.4f, valence : %.4f \n' % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0])
def genetic_algorithm(mutation=0.1, population=10, function="rosenbrock", max_iterations=5000, target_fitness=None, dimensions=2, selection="elitism"): """ :param crossover: :param mutation: :param selection: :param pop_size: :return: - init agents x at random positions in search space - calculate fitness value of each agent - while(not end) : (end can be number of iterations, adequate fitness reached, ...) - for each agent : - calculate whether selected (selection strategy) - for each agent : - if selected, use as parent - otherwise, calculate cross-over with parents based on selection and mutation - calculate fitness value in the new iteration - Pick agent with best fitness and return it as solution """ bounds = [-2.4, 2.4] if function == "rosenbrock" else [-5.12, 5.12] agents = [[ np.random.uniform(low=bounds[0], high=bounds[1]) for _ in range(dimensions) ] for _ in range(population)] for agent in agents: np.clip(agent, bounds[0], bounds[1]) end = False iteration = 1 best_fitness = None first_run = True agents_history = np.zeros((max_iterations, len(agents), 2), dtype=float) fitness_history = np.zeros((max_iterations, len(agents)), dtype=float) fitness_history[0] = [evaluate(function, agent) for agent in agents] num_selected = int( population / 5) if selection == "elitism" or selection == "roulette" else int( population / 5 * 4) if selection == "steady" else int(population / 2) while not end: iteration_history = [] selected_agents = [] # SELECTION if selection == "elitism" or selection == "steady": selected_agents = np.argpartition(fitness_history[iteration - 1], num_selected + 1) elif selection == "tournament": random_order = np.random.choice(range(population), population, replace=False) left_bracket = random_order[:num_selected] right_bracket = random_order[num_selected:] for i in range(num_selected): selected_agent = left_bracket[i] if fitness_history[ iteration - 1][left_bracket[i]] > fitness_history[ iteration - 1][right_bracket[i]] else right_bracket[ i] if fitness_history[iteration - 1][ left_bracket[i]] < fitness_history[ iteration - 1][right_bracket[i]] else np.random.choice( [left_bracket[i], right_bracket[i]]) selected_agents.append(selected_agent) elif selection == "roulette": total_fitness = np.sum(fitness_history[iteration - 1]) random_order = np.random.choice(range(population), population, replace=False) roulette_selection = { int(i): fitness_history[iteration - 1][int(i)] for i in random_order } chance = np.random.uniform(0, total_fitness) i = 0 current = 0 for key, value in roulette_selection.items(): current += value if current > chance: selected_agents.append(key) i += 1 if i >= num_selected: break else: # in case of error, default to elitism selected_agents = np.argpartition(fitness_history[iteration - 1], num_selected + 1) if first_run: first_run = False iteration = 0 for x in range(len(agents)): if x in selected_agents[:num_selected]: new_agent = agents[x] else: # CROSSOVER if len(selected_agents) > 0: dad = np.random.choice(selected_agents) mom = np.random.choice(selected_agents) while dad == mom: mom = np.random.choice(selected_agents) else: # Roulette selection is special parents = np.random.choice(range(population), 2, replace=False) dad = parents[0] mom = parents[1] new_agent = cross_over(agents[dad], agents[mom]) # MUTATION if np.random.rand() < mutation: new_agent = mutate(new_agent, bounds[0], bounds[1]) agents[x] = new_agent iteration_history.append(new_agent) agents_history[iteration] = np.clip(iteration_history, bounds[0], bounds[1]) fitness_history[iteration] = [ evaluate(function, agent) for agent in agents ] iteration += 1 if (max_iterations != None) and (max_iterations == iteration): end = True if (target_fitness != None) and (target_fitness == best_fitness): end = True # calculate total fitness for all agents fitnesses = np.array([[evaluate(function, agent) for agent in agents] for agents in agents_history]) best_agent_id = int(np.argmin(fitnesses) / max_iterations) % population best_agent = agents_history[:, best_agent_id, :] best_fitness = fitnesses[:, best_agent_id] return best_agent, best_fitness, agents_history #best_fitness_history
i, :].argsort()[-args.re_ranking:][::-1].tolist() _, activations_2nd = capsule_net(data[i, :].unsqueeze(0), candidate_labels) Y_pred[i, candidate_labels] = activations_2nd.squeeze( 2).data.cpu().numpy() for i in range(Y_pred.shape[0]): sorted_idx = np.argpartition(-Y_pred[i, :], top_k)[:top_k] row_idx_list += [i + start_idx] * top_k col_idx_list += (sorted_idx).tolist() val_idx_list += Y_pred[i, sorted_idx].tolist() done = time.time() elapsed = done - start print("\r Reranking: {} Iteration: {}/{} ({:.1f}%) Loss: {:.5f} {:.5f}". format(args.re_ranking, batch_idx, nr_batches, batch_idx * 100 / nr_batches, 0, elapsed), end="") m = max(row_idx_list) + 1 n = max(k_trn, k_tst) print(elapsed) Y_tst_pred = sp.csr_matrix((val_idx_list, (row_idx_list, col_idx_list)), shape=(m, n)) if k_trn >= k_tst: Y_tst_pred = Y_tst_pred[:, :k_tst] evaluate(Y_tst_pred.toarray(), Y_tst)
def rnn_cv( folds, n_hidden=10, n_epochs=50, lr=0.001, lrd = 0.999, reg_coef= 0.01, doSmoothing=False, useEssentia=False): doSaveModel = False if doSmoothing: dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g_smoothed'%(nb_features, n_hidden, n_epochs, lr, reg_coef) else: dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g'%(nb_features, n_hidden, n_epochs, lr, reg_coef) MODELDIR = 'rnn/' + dir_name + '/' LOGDIR = MODELDIR if not path.exists(MODELDIR): makedirs(MODELDIR) print '... output dir: %s'%(MODELDIR) # smoothing params taille = 12 wts = np.ones(taille-1)*1./taille wts = np.hstack((np.array([1./(2*taille)]), wts, np.array([1./(2*taille)]))) delay = (wts.shape[0]-1) / 2 # # initialize global logger variable # print '... initializing global logger variable' # logger = logging.getLogger(__name__) # withFile = False # logger = settings.init(MODELDIR + 'train.log', withFile) # perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef) perf_file_name = LOGDIR + 'performance.log' log_f = open(perf_file_name, 'w') all_fold_pred = list() all_fold_y_test = list() all_fold_id_test = list() for fold_id in range(10): # fold_id = 0 fold = folds[fold_id] t0 = time.time() # print '... loading FOLD %d'%fold_id # if useEssentia: # fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed_essentia.pkl'%(fold_id), "rb" ) ) if useEssentia: X_train = fold['train']['X'] y_train = fold['train']['y'] id_train = fold['train']['song_id'] X_test = fold['test']['X'] y_test = fold['test']['y'] id_test = fold['test']['song_id'] else: fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) ) X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT) X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT) print X_train.shape, y_train.shape, X_test.shape, y_test.shape if useMelodyFeatures: # first feature = slope, other two = mean, std melody_train, melody_test = subset_features(all_song_melody_features, id_train, id_test) # melody_train = melody_train[:,:,1:] # melody_test = melody_test[:,:,1:] # standardize train data melody_concat_train = np.reshape(melody_train, (melody_train.shape[0]*melody_train.shape[1], melody_train.shape[2]), order='C') melody_concat_train_normed, scaler = standardize(melody_concat_train) # print concat_train_normed.shape melody_train_normed = np.reshape(melody_concat_train_normed, (melody_train.shape[0], melody_train.shape[1], melody_train.shape[2]), order='C') del melody_concat_train, melody_concat_train_normed # standardize test data melody_concat_test = np.reshape(melody_test, (melody_test.shape[0]*melody_test.shape[1], melody_test.shape[2]), order='C') melody_concat_test_normed, _ = standardize(melody_concat_test, scaler) # print concat_test_normed.shape melody_test_normed = np.reshape(melody_concat_test_normed, (melody_test.shape[0], melody_test.shape[1], melody_test.shape[2]), order='C') del melody_concat_test, melody_concat_test_normed # concat with the other features X_train = np.concatenate((X_train, melody_train_normed), axis=2) X_test = np.concatenate((X_test, melody_test_normed), axis=2) if useTempoFeatures: tempo_train, tempo_test = subset_features(all_song_tempo_features, id_train, id_test) # standardize train data tempo_concat_train = np.reshape(tempo_train, (tempo_train.shape[0]*tempo_train.shape[1], tempo_train.shape[2]), order='C') tempo_concat_train_normed, scaler = standardize(tempo_concat_train) # print concat_train_normed.shape tempo_train_normed = np.reshape(tempo_concat_train_normed, (tempo_train.shape[0], tempo_train.shape[1], tempo_train.shape[2]), order='C') del tempo_concat_train, tempo_concat_train_normed # standardize test data tempo_concat_test = np.reshape(tempo_test, (tempo_test.shape[0]*tempo_test.shape[1], tempo_test.shape[2]), order='C') tempo_concat_test_normed, _ = standardize(tempo_concat_test, scaler) # print concat_test_normed.shape tempo_test_normed = np.reshape(tempo_concat_test_normed, (tempo_test.shape[0], tempo_test.shape[1], tempo_test.shape[2]), order='C') del tempo_concat_test, tempo_concat_test_normed # concat with the other features X_train = np.concatenate((X_train, tempo_train_normed), axis=2) X_test = np.concatenate((X_test, tempo_test_normed), axis=2) # print id_test.shape # X_train = X_train[0:100,:,:] # y_train = y_train[0:100,:,:] # X_train = X_train[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]] # X_test = X_test[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]] # X_train = X_train[:,[13,85,103,142,214]] # X_test = X_test[:,[13,85,103,142,214]] # X_test = X_train[119:119+y_test.shape[0],:] # y_test = y_train[119:119+y_test.shape[0]] print X_train.shape, y_train.shape, X_test.shape, y_test.shape nb_seq_train, nb_frames_train, nb_features_train = X_train.shape nb_seq_test, nb_frames_test, nb_features_test = X_test.shape assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST' assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST' dim_ouput_train = y_train.shape[2] dim_ouput_test = y_test.shape[2] assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST' n_in = nb_features_train n_out = dim_ouput_train n_steps = nb_frames_train validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out, learning_rate=lr, learning_rate_decay=lrd, L1_reg=reg_coef, L2_reg=reg_coef, n_epochs=n_epochs, activation='tanh') model.fit(X_train, y_train, validation_frequency=validation_frequency) if doSaveModel: # model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef) model_name = MODELDIR + 'model_fold%d.pkl'%(fold_id) model.save(fpath=model_name) pred = list() for ind_seq_test in xrange(nb_seq_test): pred.append(model.predict(X_test[ind_seq_test])) y_hat = np.array(pred, dtype=float) print y_hat.shape if doSmoothing: # smooooooth y_hat_smooth = np.zeros_like(y_hat, dtype=float) for i in xrange(y_hat.shape[0]): y_hat_smooth[i, :, 0] = np.convolve(y_hat[i, :, 0], wts, mode='same') y_hat_smooth[i, :delay, 0] = y_hat[i, :delay, 0] y_hat_smooth[i, -delay:, 0] = y_hat[i, -delay:, 0] y_hat_smooth[i, :, 1] = np.convolve(y_hat[i, :, 1], wts, mode='same') y_hat_smooth[i, :delay, 1] = y_hat[i, :delay, 1] y_hat_smooth[i, -delay:, 1] = y_hat[i, -delay:, 1] # save predictions on the test subset, before reshaping to 2-d arrays (I need 3d arrays) if doSmoothing: # fold_pred = [item for sublist in fold_pred for item in sublist] # fold_pred = np.array(fold_pred, dtype=float) pred_file = LOGDIR + 'fold%d_test_predictions.pkl'%(fold_id) pickle.dump( y_hat_smooth, open( pred_file, "wb" ) ) print ' ... predictions y_hat_smooth saved in: %s'%(pred_file) else: # fold_pred = [item for sublist in fold_pred for item in sublist] # fold_pred = np.array(fold_pred, dtype=float) pred_file = LOGDIR + 'fold%d_test_predictions.pkl'%(fold_id) pickle.dump( y_hat, open( pred_file, "wb" ) ) print ' ... predictions y_hat saved in: %s'%(pred_file) if doSmoothing: y_hat_smooth = np.reshape(y_hat_smooth, (y_hat_smooth.shape[0]*y_hat_smooth.shape[1], y_hat_smooth.shape[2])) y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2])) y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2])) print y_hat.shape, y_test_concat.shape assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!' # concat hyp labels: if doSmoothing: all_fold_pred.append(y_hat_smooth.tolist()) else: all_fold_pred.append(y_hat.tolist()) # concat ref labels: all_fold_y_test.append(y_test_concat.tolist()) if doSmoothing: RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat_smooth, id_test.shape[0]) else: RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0]) s = ( 'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n' % (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]) ) print s log_f.write(s) # predict on the train set and save predictions (useful to train rnn2) if doSmoothing: pred = list() for ind_seq_train in xrange(nb_seq_train): pred.append(model.predict(X_train[ind_seq_train])) train_y_hat = np.array(pred, dtype=float) print train_y_hat.shape train_y_hat_smooth = np.zeros_like(train_y_hat, dtype=float) for i in xrange(train_y_hat.shape[0]): train_y_hat_smooth[i, :, 0] = np.convolve(train_y_hat[i, :, 0], wts, mode='same') train_y_hat_smooth[i, :delay, 0] = train_y_hat[i, :delay, 0] train_y_hat_smooth[i, -delay:, 0] = train_y_hat[i, -delay:, 0] train_y_hat_smooth[i, :, 1] = np.convolve(train_y_hat[i, :, 1], wts, mode='same') train_y_hat_smooth[i, :delay, 1] = train_y_hat[i, :delay, 1] train_y_hat_smooth[i, -delay:, 1] = train_y_hat[i, -delay:, 1] # no reshape, I need 3d arrays # train_y_hat_smooth = np.reshape(train_y_hat_smooth, (train_y_hat_smooth.shape[0]*train_y_hat_smooth.shape[1], train_y_hat_smooth.shape[2])) pred_file = LOGDIR + 'fold%d_train_predictions.pkl'%(fold_id) pickle.dump( train_y_hat_smooth, open( pred_file, "wb" ) ) print ' ... predictions y_hat_smooth saved in: %s'%(pred_file) else: pred = list() for ind_seq_train in xrange(nb_seq_train): pred.append(model.predict(X_train[ind_seq_train])) train_y_hat = np.array(pred, dtype=float) pred_file = LOGDIR + 'fold%d_train_predictions.pkl'%(fold_id) pickle.dump( train_y_hat, open( pred_file, "wb" ) ) print ' ... predictions y_hat saved in: %s'%(pred_file) doPlot = False if doPlot: fig, ax = plt.subplots() x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0]) if EMO == 'valence': ax.plot(x1, y_test_concat[:, 0], 'o', label="Data") # ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction") ax.plot(x1, y_hat[:,0], 'ro', label="OLS prediction") else: ax.plot(x1, y_test_concat[:, 1], 'o', label="Data") ax.plot(x1, y_hat[:,1], 'ro', label="OLS prediction") plt.title(EMO + ' on Test subset') ax.legend(loc="best") plt.show() # plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png') doPlotTrain = False if doPlotTrain: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_train[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_train[0]) guess = model.predict(X_train[0]) guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() doPlotTest = False if doPlotTest: # plt.close('all') fig = plt.figure() ax1 = plt.subplot(211) plt.plot(X_test[0]) ax1.set_title('input') ax2 = plt.subplot(212) true_targets = plt.plot(y_test[0]) # guess = model.predict(X_test[0]) guess = y_hat[0] guessed_targets = plt.plot(guess, linestyle='--') for i, x in enumerate(guessed_targets): x.set_color(true_targets[i].get_color()) ax2.set_title('solid: true output, dashed: model output') plt.show() print "... Elapsed time: %f" % (time.time() - t0) all_fold_pred = [item for sublist in all_fold_pred for item in sublist] all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist] all_fold_pred = np.array(all_fold_pred, dtype=float) all_fold_y_test = np.array(all_fold_y_test, dtype=float) print all_fold_pred.shape, all_fold_y_test.shape # save predictions pred_file = LOGDIR + 'all_predictions.pkl' pickle.dump( all_fold_pred, open( pred_file, "wb" ) ) print ' ... all predictions saved in: %s'%(pred_file) # ref_file = 'rnn/all_groundtruth.pkl' # pickle.dump( all_fold_y_test, open( ref_file, "wb" ) ) # compute t-test p-values with baseline predictions baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl' baseline_preds = pickle.load(open( baseline_prediction_file, 'r' )) pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1] pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1] pvalues = (pvalue_val, pvalue_ar) RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0) # print( # 'sklearn --> valence: %.4f, arousal: %.4f\n' # 'Pearson Corr --> valence: %.4f, arousal: %.4f \n' # # % (RMSE[0], -1. , pcorr[0][0], -1) # % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0]) # ) s = ( 'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n' % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar) ) print s log_f.write(s) log_f.close() return RMSE, pcorr, pvalues
def validate(val_loader, net, criterion, optimizer, epoch, train_args, restore, visualize): net.eval() val_loss = AverageMeter() inputs_all, gts_all, predictions_all = [], [], [] for vi, data in enumerate(val_loader): inputs, gts = data N = inputs.size(0) inputs = Variable(inputs, volatile=True).cuda() gts = Variable(gts, volatile=True).cuda() outputs = net(inputs) predictions = outputs.data.max(1)[1].squeeze_(1).squeeze_( 0).cpu().numpy() val_loss.update(criterion(outputs, gts).data[0] / N, N) if random.random() > train_args['val_img_sample_rate']: inputs_all.append(None) else: inputs_all.append(inputs.data.squeeze_(0).cpu()) gts_all.append(gts.data.squeeze_(0).cpu().numpy()) predictions_all.append(predictions) acc, acc_cls, mean_iu, fwavacc = evaluate(predictions_all, gts_all, voc.num_classes) if mean_iu > train_args['best_record']['mean_iu']: train_args['best_record']['val_loss'] = val_loss.avg train_args['best_record']['epoch'] = epoch train_args['best_record']['acc'] = acc train_args['best_record']['acc_cls'] = acc_cls train_args['best_record']['mean_iu'] = mean_iu train_args['best_record']['fwavacc'] = fwavacc snapshot_name = 'epoch_%d_loss_%.5f_acc_%.5f_acc-cls_%.5f_mean-iu_%.5f_fwavacc_%.5f_lr_%.10f' % ( epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc, optimizer.param_groups[1]['lr']) # torch.save(net.state_dict(), os.path.join(ckpt_path, exp_name, snapshot_name + '.pth')) # torch.save(optimizer.state_dict(), os.path.join(ckpt_path, exp_name, 'opt_' + snapshot_name + '.pth')) if train_args['val_save_to_img_file']: to_save_dir = os.path.join(ckpt_path, exp_name, str(epoch)) check_mkdir(to_save_dir) val_visual = [] for idx, data in enumerate(zip(inputs_all, gts_all, predictions_all)): if data[0] is None: continue input_pil = restore(data[0]) gt_pil = voc.colorize_mask(data[1]) predictions_pil = voc.colorize_mask(data[2]) if train_args['val_save_to_img_file']: input_pil.save(os.path.join(to_save_dir, '%d_input.png' % idx)) predictions_pil.save( os.path.join(to_save_dir, '%d_prediction.png' % idx)) gt_pil.save(os.path.join(to_save_dir, '%d_gt.png' % idx)) val_visual.extend([ visualize(input_pil.convert('RGB')), visualize(gt_pil.convert('RGB')), visualize(predictions_pil.convert('RGB')) ]) val_visual = torch.stack(val_visual, 0) val_visual = vutils.make_grid(val_visual, nrow=3, padding=5) writer.add_image(snapshot_name, val_visual) print( '--------------------------------------------------------------------') print( '[epoch %d], [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f]' % (epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc)) print( 'best record: [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f], [epoch %d]' % (train_args['best_record']['val_loss'], train_args['best_record']['acc'], train_args['best_record']['acc_cls'], train_args['best_record']['mean_iu'], train_args['best_record']['fwavacc'], train_args['best_record']['epoch'])) print( '--------------------------------------------------------------------') writer.add_scalar('val_loss', val_loss.avg, epoch) writer.add_scalar('acc', acc, epoch) writer.add_scalar('acc_cls', acc_cls, epoch) writer.add_scalar('mean_iu', mean_iu, epoch) writer.add_scalar('fwavacc', fwavacc, epoch) writer.add_scalar('lr', optimizer.param_groups[1]['lr'], epoch) net.train() return val_loss.avg
# 1. train model with training data, and recommend samples for labeling out = AL.process(X_train, y_train, X_test, Nrecs, 'SVM') IDtoLabel = out['IDtoLabel'] relevanceScore = out['relevanceScore'] # 2. assume get the expert feedback y_newlabel = y_test[IDtoLabel] # For Simulation purpose # performance evaluation: calculate prediction accuracy classifier = out['classifier'] y_test_predicted = classifier.predict(X_test) out = evaluate(y_test, y_test_predicted) accuracy.append(out['accuracy']) # updated y_test mask = np.ones(len(y_test), dtype=bool) # all elements included/True. mask[IDtoLabel] = False y_test = y_test[mask] # 3. update training data set for re-train model out = AL.add_answers(X_train, X_test, y_train, y_newlabel, IDtoLabel) X_train = out['X_train'] y_train = out['y_train'] X_test = out['X_test']
def main(args): device = torch.device(args.device if torch.cuda.is_available() else "cpu") print(args) print( 'Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/' ) tb_writer = SummaryWriter() if os.path.exists("./weights") is False: os.makedirs("./weights") train_images_path, train_images_label, val_images_path, val_images_label = read_split_data( args.data_path) img_size = { "s": [300, 384], # train_size, val_size "m": [384, 480], "l": [384, 480] } num_model = "s" data_transform = { "train": transforms.Compose([ transforms.RandomResizedCrop(img_size[num_model][0]), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]), "val": transforms.Compose([ transforms.Resize(img_size[num_model][1]), transforms.CenterCrop(img_size[num_model][1]), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]) } # 实例化训练数据集 train_dataset = MyDataSet(images_path=train_images_path, images_class=train_images_label, transform=data_transform["train"]) # 实例化验证数据集 val_dataset = MyDataSet(images_path=val_images_path, images_class=val_images_label, transform=data_transform["val"]) batch_size = args.batch_size nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]) # number of workers print('Using {} dataloader workers every process'.format(nw)) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=batch_size, shuffle=True, pin_memory=True, num_workers=nw, collate_fn=train_dataset.collate_fn) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False, pin_memory=True, num_workers=nw, collate_fn=val_dataset.collate_fn) # 如果存在预训练权重则载入 model = create_model(num_classes=args.num_classes).to(device) if args.weights != "": if os.path.exists(args.weights): weights_dict = torch.load(args.weights, map_location=device) load_weights_dict = { k: v for k, v in weights_dict.items() if model.state_dict()[k].numel() == v.numel() } print(model.load_state_dict(load_weights_dict, strict=False)) else: raise FileNotFoundError("not found weights file: {}".format( args.weights)) # 是否冻结权重 if args.freeze_layers: for name, para in model.named_parameters(): # 除head外,其他权重全部冻结 if "head" not in name: para.requires_grad_(False) else: print("training {}".format(name)) pg = [p for p in model.parameters() if p.requires_grad] optimizer = optim.SGD(pg, lr=args.lr, momentum=0.9, weight_decay=1E-4) # Scheduler https://arxiv.org/pdf/1812.01187.pdf lf = lambda x: ((1 + math.cos(x * math.pi / args.epochs)) / 2) * ( 1 - args.lrf) + args.lrf # cosine scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) for epoch in range(args.epochs): # train train_loss, train_acc = train_one_epoch(model=model, optimizer=optimizer, data_loader=train_loader, device=device, epoch=epoch) scheduler.step() # validate val_loss, val_acc = evaluate(model=model, data_loader=val_loader, device=device, epoch=epoch) tags = [ "train_loss", "train_acc", "val_loss", "val_acc", "learning_rate" ] tb_writer.add_scalar(tags[0], train_loss, epoch) tb_writer.add_scalar(tags[1], train_acc, epoch) tb_writer.add_scalar(tags[2], val_loss, epoch) tb_writer.add_scalar(tags[3], val_acc, epoch) tb_writer.add_scalar(tags[4], optimizer.param_groups[0]["lr"], epoch) torch.save(model.state_dict(), "./weights/model-{}.pth".format(epoch))
def validate(val_loader, net, criterion, optimizer, epoch, train_args, visualize): # the following code is written assuming that batch size is 1 net.eval() val_loss = AverageMeter() gts_all = np.zeros((len(val_loader), args['shorter_size'], 2 * args['shorter_size']), dtype=int) predictions_all = np.zeros((len(val_loader), args['shorter_size'], 2 * args['shorter_size']), dtype=int) for vi, data in enumerate(val_loader): input, gt, slices_info = data assert len(input.size()) == 5 and len(gt.size()) == 4 and len(slices_info.size()) == 3 input.transpose_(0, 1) gt.transpose_(0, 1) slices_info.squeeze_(0) assert input.size()[3:] == gt.size()[2:] count = torch.zeros(args['shorter_size'], 2 * args['shorter_size']).cuda() output = torch.zeros(voc.num_classes, args['shorter_size'], 2 * args['shorter_size']).cuda() slice_batch_pixel_size = input.size(1) * input.size(3) * input.size(4) for input_slice, gt_slice, info in zip(input, gt, slices_info): input_slice = Variable(input_slice).cuda() gt_slice = Variable(gt_slice).cuda() output_slice = net(input_slice) assert output_slice.size()[2:] == gt_slice.size()[1:] assert output_slice.size()[1] == voc.num_classes output[:, info[0]: info[1], info[2]: info[3]] += output_slice[0, :, :info[4], :info[5]].data gts_all[vi, info[0]: info[1], info[2]: info[3]] += gt_slice[0, :info[4], :info[5]].data.cpu().numpy() count[info[0]: info[1], info[2]: info[3]] += 1 val_loss.update(criterion(output_slice, gt_slice).data[0], slice_batch_pixel_size) output /= count gts_all[vi, :, :] /= count.cpu().numpy().astype(int) predictions_all[vi, :, :] = output.max(0)[1].squeeze_(0).cpu().numpy() print('validating: %d / %d' % (vi + 1, len(val_loader))) acc, acc_cls, mean_iu, fwavacc = evaluate(predictions_all, gts_all, voc.num_classes) train_args['best_record']['val_loss'] = val_loss.avg train_args['best_record']['epoch'] = epoch train_args['best_record']['acc'] = acc train_args['best_record']['acc_cls'] = acc_cls train_args['best_record']['mean_iu'] = mean_iu train_args['best_record']['fwavacc'] = fwavacc snapshot_name = 'epoch_%d_loss_%.5f_acc_%.5f_acc-cls_%.5f_mean-iu_%.5f_fwavacc_%.5f_lr_%.10f' % ( epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc, optimizer.param_groups[1]['lr']) torch.save(net.state_dict(), os.path.join(ckpt_path, exp_name, snapshot_name + '.pth')) torch.save(optimizer.state_dict(), os.path.join(ckpt_path, exp_name, 'opt_' + snapshot_name + '.pth')) if train_args['val_save_to_img_file']: to_save_dir = os.path.join(ckpt_path, exp_name, str(epoch)) check_mkdir(to_save_dir) val_visual = [] for idx, data in enumerate(zip(gts_all, predictions_all)): gt_pil = voc.colorize_mask(data[0]) predictions_pil = voc.colorize_mask(data[1]) if train_args['val_save_to_img_file']: predictions_pil.save(os.path.join(to_save_dir, '%d_prediction.png' % idx)) gt_pil.save(os.path.join(to_save_dir, '%d_gt.png' % idx)) val_visual.extend([visualize(gt_pil.convert('RGB')), visualize(predictions_pil.convert('RGB'))]) val_visual = torch.stack(val_visual, 0) val_visual = vutils.make_grid(val_visual, nrow=2, padding=5) writer.add_image(snapshot_name, val_visual) print('-----------------------------------------------------------------------------------------------------------') print('[epoch %d], [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f]' % ( epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc)) print('best record: [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f], [epoch %d]' % ( train_args['best_record']['val_loss'], train_args['best_record']['acc'], train_args['best_record']['acc_cls'], train_args['best_record']['mean_iu'], train_args['best_record']['fwavacc'], train_args['best_record']['epoch'])) print('-----------------------------------------------------------------------------------------------------------') writer.add_scalar('val_loss', val_loss.avg, epoch) writer.add_scalar('acc', acc, epoch) writer.add_scalar('acc_cls', acc_cls, epoch) writer.add_scalar('mean_iu', mean_iu, epoch) writer.add_scalar('fwavacc', fwavacc, epoch) net.train() return val_loss.avg
def run_latency_per_round(train_loader, test_loader, comm_matrix, num_rounds, epochs, num_clients, latency_nodes, latency_rounds, net='net', optimizer='sgd', lr=0.1): """ Runs a decentralized optimization algorithm for the given learning rate for a number of rounds, over some network. Some nodes send their weights with a one-rounds latency, only during specific rounds. Outputs the accuracies and returns them. Params: train_loader (array): the list of all train datasets, one per client test_loader (array): the list of test datasets, one per client comm_matrix (numpy.array): the communication matric modeling the network num_rounds (int): the number of data exchanges between nodes epochs (int): the number of optimization steps between each communication (minimum 1) num_clients (int): the number of clients in the network latency_nodes (array): the list of delayed nodes latency_rounds (array): the rounds at which latency will occur across the network net (string): the neural network framework we use optimizer (string): the chosen optimizer, SGD by default lr (double): the learning rate for the optimizaion algorithm Returns: global_model (nn.Module): the final global neural network averaging all the clients client_models (array of Net): the list of all the final client neural networks accs (array): the corresponding accuracies, with the same shape as lrs """ accs = [] global_model, client_models = model_init(num_clients, net) opt = optimizer_init(client_models, lr, optimizer) loss, test_loss, acc = 0.0, 0.0, 0.0 for r in range(num_rounds): old_client_models = client_models # client update for i in range(num_clients): loss += client_update(client_models[i], opt[i], train_loader[i], epoch=epochs) # diffuse params if (r in latency_rounds): diffuse_params_latency(client_models, comm_matrix, latency_nodes) print("round {}, delay".format(r)) elif (r in latency_rounds + 1): diffuse_params(client_models, comm_matrix) diffuse_params_latency( old_client_models, comm_matrix, np.setdiff1d(np.array(range(num_clients)), latency_nodes)) print("round {}, delay recovery".format(r)) else: diffuse_params(client_models, comm_matrix) print("round {}, normal".format(r)) average_models(global_model, client_models) test_loss, acc = evaluate(global_model, test_loader) print('%d-th round' % r) print('average train loss %0.3g | test loss %0.3g | test acc: %0.3f' % (loss / num_clients, test_loss, acc)) accs.append(acc) return global_model, client_models, accs