def CDNF(): t = 0 if EQ(True) == True: return True else: u = EQ(True) label .zero t = t + 1 (H[t], S[t], a[t]) = (False, None, u) label .one if EQ(invjoin(H, 'and')) == True: return invjoin(H, 'and') else: u = EQ(invjoin(H, 'and')) I = [] for i in range(0, len(H)): if evaluate(u, H[i])==False: I.append(i) if I==[]: goto .zero mu = [] for i in I: mu.append(walk(u, a[i])) S.append(mu[-1] ^ a[i]) for i in range(0,t): H[i] = MDNF(S[i]) goto .one
def make_move(self, request):
"""Makes a move. Returns a game state with message. """
game = get_by_urlsafe(request.urlsafe_game_key, Game)
#check if the game is over, update the score list if needed
if evaluate(game.state):
outcome = evaluate(game.state)
if outcome == "O":
msg = "Win"
else:
msg = "Lose"
game.end_game(msg)
game.game_over = True
game.put()
else:
if "-" not in game.state:
game.end_game("Draw")
game.game_over = True
if game.game_over:
return game.to_form('Game already over!')
#check if it is player's turn
if game.player:
if request.move not in range(0,9):
raise endpoints.BadRequestException('Invalid Move')
if game.state[request.move] != "-":
raise endpoints.BadRequestException('Invalid Move')
#update the board state
board = list(game.state)
board[request.move] = "O"
game.state = ''.join(board)
#update movecount
game.movecount += 1
#evaluate the result
if evaluate(game.state):
game.end_game("Win")
return game.to_form('You win!')
if "-" not in game.state:
game.end_game("Draw")
return game.to_form('This is a Draw.')
msg = "AI's turn"
game.history()
game.player = False
game.put()
return game.to_form(msg)
else:
msg = "This is not your turn"
return game.to_form(msg)
def get_random_move(self,request):
"""Get Random Generated Moves"""
game = get_by_urlsafe(request.urlsafe_game_key, Game)
#check if the game is over, update the score list if needed
if evaluate(game.state):
outcome = evaluate(game.state)
if outcome == "O":
msg = "Win"
game.end_game(msg)
else:
msg = "Lose"
game.end_game(msg)
game.game_over = True
game.put()
else:
if "-" not in game.state:
game.end_game("Draw")
game.game_over = True
if game.game_over:
return game.to_form('Game already over!')
#check if it is ai's turn
if game.player:
msg = "Waiting for your move."
return game.to_form(msg)
else:
#update board with a random move
game.state = add_random_move(game.state)
#update movecount
game.movecount += 1
#evaluate the result
if evaluate(game.state):
game.end_game("Lose")
return game.to_form('You Lose!')
if "-" not in game.state:
game.end_game("Draw")
return game.to_form('This is a Draw.')
msg = "Your turn"
game.history()
game.player = True
game.put()
return game.to_form(msg)
def evaluate_model(self, k, max_warping_window, train_data, train_label, test_data, test_label):
print '--------------------------'
print '--------------------------\n'
print 'Running for k = ', k
print 'Running for w = ', max_warping_window
model = KnnDtw(k_neighbours = k, max_warping_window = max_warping_window)
model.fit(train_data, train_label)
predicted_label, probability = model.predict(test_data)
print '\nPredicted : ', predicted_label
print 'Actual : ', test_label
accuracy, precision, recall, f1score = evaluate(self.labels, predicted_label, test_label)
print 'Avg/Total Accuracy :', accuracy
print 'Avg/Total Precision :', precision
print 'Avg/Total Recall :', recall
print 'Avg/Total F1 Score :', f1score
if USE_CUDA and args.model:
print("USING CUDA")
m = m.cuda()
return m
if args.use:
model = get_model()
model.load_state_dict(torch.load(args.use))
for fold, (train, val) in enumerate(train_val_generator):
print("FOLD " + str(fold))
train_iter, val_iter, test_iter = data_handler.get_bucket_iterators(
(train, val, test), args.batch_size)
if args.model == 'CNN' or args.model == 'CNNMulti' or args.model == 'LSTM':
p, r, f1, p1, r1, f11 = utils.evaluate(model, val_iter)
elif args.model == 'CNNFeatures' or args.model == 'CNNMultiFeatures' or args.model == 'LSTMFeatures':
p, r, f1, p1, r1, f11 = utils.evaluate(model,
val_iter,
has_features=True)
p_avg += p
r_avg += r
f1_avg += f1
p1_avg += p1
r1_avg += r1
f11_avg += f11
if args.model == 'CNN' or args.model == 'CNNMulti' or args.model == 'LSTM':
p, r, f1, p1, r1, f11 = utils.evaluate(model, test_iter)
elif args.model == "CNNFeatures" or args.model == 'CNNMultiFeatures' or args.model == 'LSTMFeatures':
p, r, f1, p1, r1, f11 = utils.evaluate(model,
CM.zero_grad()
RM.zero_grad()
loss.backward()
CM_optim.step()
RM_optim.step()
if (episode + 1) % 100 == 0:
print("episode:", episode + 1, "loss %.4f" % loss.item())
if (episode + 1) % 2000 == 0:
# test
print("Testing...")
gzsl_unseen_accuracy = evaluate(test_features, test_label,
np.arange(32), attributes, CM, RM, GPU)
gzsl_seen_accuracy = evaluate(test_seen_features, test_seen_label,
np.arange(32), attributes, CM, RM, GPU)
H = 2 * gzsl_seen_accuracy * gzsl_unseen_accuracy / (
gzsl_unseen_accuracy + gzsl_seen_accuracy)
print('seen=%.4f, unseen=%.4f, h=%.4f' %
(gzsl_seen_accuracy, gzsl_unseen_accuracy, H))
if H > last_H:
# save networks
torch.save(CM, "./models/APY_CM.pkl")
torch.save(RM, "./models/APY_RM.pkl")
def serve(self):
value = self.value.serve()
return round(evaluate(self.function, value), 5)
print('error', user_id)
elif label_sum == 1:
useful_recall.append(g)
df_useful_recall = pd.concat(useful_recall, sort=False)
log.debug(f'df_useful_recall: {df_useful_recall.head()}')
df_useful_recall = df_useful_recall.sort_values(
['user_id', 'sim_score'], ascending=[True,
False]).reset_index(drop=True)
# 计算相关指标
if mode == 'valid':
total = df_query[df_query['click_article_id'] != -1].user_id.nunique()
hitrate_5_1, mrr_5_1, hitrate_10_1, mrr_10_1, hitrate_20_1, mrr_20_1, hitrate_40_1, mrr_40_1, hitrate_50_1, mrr_50_1 = evaluate(
recall_final[recall_final['label'].notnull()], total)
log.debug(
f'召回合并后,未删除无用用户指标: {hitrate_5_1}, {mrr_5_1}, {hitrate_10_1}, {mrr_10_1}, {hitrate_20_1}, {mrr_20_1}, {hitrate_40_1}, {mrr_40_1}, {hitrate_50_1}, {mrr_50_1}'
)
hitrate_5, mrr_5, hitrate_10, mrr_10, hitrate_20, mrr_20, hitrate_40, mrr_40, hitrate_50, mrr_50 = evaluate(
df_useful_recall[df_useful_recall['label'].notnull()], total)
log.debug(
f'召回合并后,删除无用用户指标: {hitrate_5}, {mrr_5}, {hitrate_10}, {mrr_10}, {hitrate_20}, {mrr_20}, {hitrate_40}, {mrr_40}, {hitrate_50}, {mrr_50}'
)
df = df_useful_recall['user_id'].value_counts().reset_index()
df.columns = ['user_id', 'cnt']
log.debug(f"删除后平均每个用户召回数量：{df['cnt'].mean()}")
def train(sampling_method, k_folds, data_dir, results_dir, device='cpu', use_international=False, verbose=True):
model = 'lgbm'
start_time = time.time()
if verbose:
print("Using device: {}".format(device))
print("Reading train data in...")
if use_international:
print("Using international class.")
X_train, Y_train, feature_labels = get_train(data_dir, one_hot=False, use_international=use_international)
categorical_feature = ['age_bucket', 'gender', 'signup_method', 'signup_flow', 'language',
'affiliate_channel', 'affiliate_provider', 'first_affiliate_tracked',
'signup_app', 'first_device_type', 'first_browser']
if verbose:
print("Successfully loaded data")
print("Starting Cross-Validation with {} folds...".format(k_folds))
kf = KFold(n_splits=k_folds)
kf.get_n_splits(X_train)
params = {
'task': 'train',
'objective': 'multiclass',
'num_class': 12,
'num_leaves': 31,
'lambda_l2': 0.1,
'learning_rate': 0.3,
'feature_fraction': 0.9,
'min_child_weight': 1.0,
'device': device,
'gpu_device_id': 0,
'gpu_platform_id': 0,
'max_bin': 63,
'verbose': 0
}
if use_international:
params['objective'] = 'binary'
del params["num_class"]
for k, (train_index, test_index) in enumerate(kf.split(X_train)):
print("Processing Fold {} out of {}".format(k+1, k_folds))
X_trainCV, X_testCV = X_train[train_index], X_train[test_index]
Y_trainCV, Y_testCV = Y_train[train_index], Y_train[test_index]
if verbose:
print("{} sampling process started...".format(sampling_method))
curr_time = time.time()
if sampling_method == "adasyn":
X_train_resampled, Y_train_resampled = ADASYN().fit_sample(X_trainCV, Y_trainCV)
elif sampling_method == "smote":
X_train_resampled, Y_train_resampled = SMOTE().fit_sample(X_trainCV, Y_trainCV)
elif sampling_method == "random":
X_train_resampled, Y_train_resampled = RandomOverSampler().fit_sample(X_trainCV, Y_trainCV)
elif sampling_method == "smoteenn":
X_train_resampled, Y_train_resampled = SMOTEENN().fit_sample(X_trainCV, Y_trainCV)
else:
X_train_resampled, Y_train_resampled = X_trainCV, Y_trainCV
if verbose:
print("Oversampling completed")
print("Time Taken: {:.2f}".format(time.time()-curr_time))
print("Size of Oversampled data: {}".format(X_train_resampled.shape))
print("{} model(s) selected for classification".format(model))
curr_time = time.time()
lgb_train = lgb.Dataset(data=X_train_resampled, label=Y_train_resampled,
feature_name=feature_labels, categorical_feature=categorical_feature)
clf = lgb.train(params, lgb_train, num_boost_round=30)
print("Time taken: {:.2f}".format(time.time()-curr_time))
Y_probs = clf.predict(X_testCV)
result = evaluate(Y_testCV, Y_probs)
print(result)
feature_imp = clf.feature_importance(importance_type='gain')
feature_imp = {label: imp for label, imp in zip(feature_labels, feature_imp)}
pickle.dump(feature_imp, open(os.path.join(results_dir, "{}_{}_feature_imp_fold_{}.p".format(model, sampling_method, k+1)), "wb" ))
pickle.dump(result, open(os.path.join(results_dir, "{}_{}_fold_{}.p".format(model, sampling_method, k+1)), "wb" ))
save_examples(X_testCV, Y_testCV, Y_probs, model, sampling_method, k+1, save_dir=results_dir)
print("Training took {:.2f}s.".format(time.time()-start_time))
print("Finished.")
comb = itertools.combinations(range(X_train.shape[0]), 2)
k = 0
Xp, yp, diff = [], [], []
for (i, j) in comb:
if y_train[i] == y_train[j] or blocks_train[i] != blocks_train[j]:
continue
Xp.append(X_train[i] - X_train[j])
diff.append(y_train[i] - y_train[j])
yp.append(np.sign(diff[-1]))
clf = svm.SVC(kernel='linear', C=C, max_iter=100000000)
clf.fit(Xp, yp)
coef = clf.coef_.ravel() / np.linalg.norm(clf.coef_)
rankings = utils.recover_rankings(tasks_test, np.dot(X_test, coef))
score = utils.evaluate(rankings, gold_test)
print 'scaler={}, deg={}, C={}: score={}'.format(scaler_name, deg, C, score)
if score > best_score:
best_score = score
best_C = C
best_coef = np.copy(coef)
best_scaler = scaler
best_scaler_name = scaler_name
best_deg = deg
print '\n\n\ndone'
print 'scaler: ', best_scaler_name
print 'deg: ', best_deg
print 'C: ', best_C
print 'coef: ', best_coef
def differential_evolution(crossover=0.9,
differential_weight=0.8,
population=10,
function="rosenbrock",
max_iterations=5000,
target_fitness=None,
dimensions=2):
# check crossover in [0,1]
# check weight in [0,2]
# check pop >= 4
# check either max_iter or target_fitness != None (I get both can be.. Untested for now)
# check dimensions > 0 and an Integer
"""
:param crossover:
:param differential_weight:
:param pop_size:
:return:
- init agents x at random positions in search space
- while(not end) : (end can be number of iterations, adequate fitness reached, ...)
- for each agent :
- pick 3 other agents at random
- pick a random index (bounded by size of problem)
- for each index :
- pick a uniformly distributed random number ri ~ U(0,1)
- if (ri < crossover) or (i = R ):
- yi = ai + F x (bi - ci)
- else :
- yi = xi
- if F(y) <= F(x) :
- replace agent x with the improved (or equal) agent y
- Pick agent with best fitness and return it as solution
"""
bounds = [-2.4, 2.4] if function == "rosenbrock" else [-5.12, 5.12]
agents = [[
np.random.uniform(low=bounds[0], high=bounds[1])
for _ in range(dimensions)
] for _ in range(population)]
for agent in agents:
np.clip(agent, bounds[0], bounds[1])
end = False
iteration = 0
best_fitness = None
agents_history = np.zeros((max_iterations, len(agents), 2), dtype=float)
while not end:
iteration_history = []
for x in range(len(agents)):
new_agent = [0 for _ in range(dimensions)]
temp = []
while len(temp) < 3:
rand_agent = random.randint(0, len(agents) - 1)
if (rand_agent not in temp) and rand_agent != x:
temp.append(rand_agent)
a = temp[0]
b = temp[1]
c = temp[2]
rand_index = 1 if dimensions < 2 else random.randint(1, dimensions)
for index in range(dimensions):
ri = np.random.uniform(0.0, 1.0)
if (ri < crossover) or (index == rand_index):
new_agent[
index] = agents[a][index] + differential_weight * (
agents[b][index] - agents[c][index])
else:
new_agent[index] = agents[x][index]
if evaluate(function, new_agent) <= evaluate(function, agents[x]):
agents[x] = new_agent
iteration_history.append(new_agent)
agents_history[iteration] = np.clip(iteration_history, bounds[0],
bounds[1])
iteration += 1
if (max_iterations != None) and (max_iterations == iteration):
end = True
if (target_fitness != None) and (target_fitness == best_fitness):
end = True
# calculate total fitness for all agents
fitnesses = np.array([[evaluate(function, agent) for agent in agents]
for agents in agents_history])
sum_fitness = [np.sum(fitnesses[:, i]) for i in range(population)]
best_agent_id = np.argmin(sum_fitness)
best_agent = agents_history[:, best_agent_id, :]
best_fitness = fitnesses[:, best_agent_id]
return best_agent, best_fitness, agents_history #best_fitness_history
if 0:
#Argument prediction
FV_arg = feature_vector.FeatureVector('argument')
train,valid = utils.create_training_and_validation_file_lists(ratio = 0.75, load=True)
Lambda2, misclassification_rates2 = train_perceptron(FV_arg, train, T_max = 20,
LR = 10.0, mode='Argument', subs_rate=0.8)
plt.plot(misclassification_rates2)
(y_hat, y) = test_perceptron(FV_arg, Lambda2, valid, mode='Argument')
errors = [1 for y1,y2 in zip(y_hat, y) if y1!=y2]
validation_error = len(errors)/float(len(y))
print (validation_error)
utils.evaluate(y, y_hat, FV_arg, mode = 'Arguments')
savedata2 = (Lambda2,misclassification_rates2)
with open('perceptron_argumentbb.data', 'wb') as f:
cPickle.dump(savedata2, f)
with open('perceptron_argument.data', 'rb') as f:
(LLambda2, misc2) = cPickle.load(f)
with open('perceptron_argument_predictionsbb.data', 'wb') as f:
cPickle.dump((y_hat, y), f)
with open('perceptron_argument_predictions.data', 'rb') as f:
(yy_hat2, yy2) = cPickle.load(f)
if 0:
#trigger prediction
all_fold_pred.append(data)
all_fold_pred = [item for sublist in all_fold_pred for item in sublist]
all_fold_pred = np.array(all_fold_pred, dtype=float)
print all_fold_pred.shape, all_fold_y_test.shape
# compute t-test p-values with baseline predictions
baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl'
baseline_preds = pickle.load(open( baseline_prediction_file, 'r' ))
pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1]
pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1]
pvalues = (pvalue_val, pvalue_ar)
RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0)
# print(
# 'sklearn --> valence: %.4f, arousal: %.4f\n'
# 'Pearson Corr --> valence: %.4f, arousal: %.4f \n'
# # % (RMSE[0], -1. , pcorr[0][0], -1)
# % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0])
# )
s = (
'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n'
% (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar)
)
print s
for ind_seq_test in xrange(nb_seq_test):
pred.append(model.predict(X_test[ind_seq_test]))
y_hat = np.array(pred, dtype=float)
y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2]))
y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2]))
print y_hat.shape, y_test_concat.shape
assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!'
all_fold_pred.append(y_hat.tolist())
all_fold_y_test.append(y_test_concat.tolist())
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0])
s = (
'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n'
% (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])
)
print s
log_f.write(s)
doPlot = False
if doPlot:
fig, ax = plt.subplots()
x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0])
if EMO == 'valence':
ax.plot(x1, y_test_concat[:, 0], 'o', label="Data")
# ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction")
def rnn_main( fold, n_hidden=10, n_epochs=100, lr=0.001, lrd = 0.999, reg_coef= 0.01):
doSaveModel = True
MODELDIR = 'AE/models/'
LOGDIR = MODELDIR
if not path.exists(MODELDIR):
makedirs(MODELDIR)
print '... output dir: %s'%(MODELDIR)
# # initialize global logger variable
# print '... initializing global logger variable'
# logger = logging.getLogger(__name__)
# withFile = False
# logger = settings.init(MODELDIR + 'train.log', withFile)
# perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef)
perf_file_name = LOGDIR + 'performance.log'
log_f = open(perf_file_name, 'w')
all_fold_pred = list()
all_fold_y_test = list()
all_fold_id_test = list()
# for fold_id in range(10):
# fold_id = 0
# fold = folds[fold_id]
t0 = time.time()
# print '... loading FOLD %d'%fold_id
# if useEssentia:
# fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed_essentia.pkl'%(fold_id), "rb" ) )
X_train = fold['train']['X']
y_train = fold['train']['y']
id_train = fold['train']['song_id']
X_test = fold['test']['X']
y_test = fold['test']['y']
id_test = fold['test']['song_id']
X_train = np.reshape(X_train, (X_train.shape[0]//60, 60, X_train.shape[1]), order='C')
X_test = np.reshape(X_test, (X_test.shape[0]//60, 60, X_test.shape[1]), order='C')
y_train = np.reshape(y_train, (y_train.shape[0]//60, 60, y_train.shape[1]), order='C')
y_test = np.reshape(y_test, (y_test.shape[0]//60, 60, y_test.shape[1]), order='C')
# else:
# # fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) )
# X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT)
# X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT)
print X_train.shape, y_train.shape, X_test.shape, y_test.shape
nb_seq_train, nb_frames_train, nb_features_train = X_train.shape
nb_seq_test, nb_frames_test, nb_features_test = X_test.shape
assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST'
assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST'
dim_ouput_train = y_train.shape[2]
dim_ouput_test = y_test.shape[2]
assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST'
n_in = nb_features_train
n_out = dim_ouput_train
n_steps = nb_frames_train
validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs
model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out,
learning_rate=lr, learning_rate_decay=lrd,
L1_reg=reg_coef, L2_reg=reg_coef,
n_epochs=n_epochs, activation='tanh')
model.fit(X_train, y_train, validation_frequency=validation_frequency)
if doSaveModel:
# model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef)
model_name = MODELDIR + 'model_fold%d.pkl'%(fold_id)
model.save(fpath=model_name)
pred = list()
for ind_seq_test in xrange(nb_seq_test):
pred.append(model.predict(X_test[ind_seq_test]))
y_hat = np.array(pred, dtype=float)
y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2]))
y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2]))
print y_hat.shape, y_test_concat.shape
assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!'
# all_fold_pred.append(y_hat.tolist())
# all_fold_y_test.append(y_test_concat.tolist())
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0])
s = (
'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n'
% (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])
)
print s
log_f.write(s)
fold_prediction_file = 'AE/pred/fold%d.pkl'%(fold_id)
pickle.dump( y_hat, open( fold_prediction_file, "wb" ) )
doPlotTrain = False
if doPlotTrain:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_train[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_train[0])
guess = model.predict(X_train[0])
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
doPlotTest = False
if doPlotTest:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_test[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_test[0])
# guess = model.predict(X_test[0])
guess = y_hat[0]
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
print "... Elapsed time: %f" % (time.time() - t0)
# all_fold_pred = [item for sublist in all_fold_pred for item in sublist]
# all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist]
#
# all_fold_pred = np.array(all_fold_pred, dtype=float)
# all_fold_y_test = np.array(all_fold_y_test, dtype=float)
#
# print all_fold_pred.shape, all_fold_y_test.shape
#
# # save predictions
# pred_file = LOGDIR + 'all_predictions.pkl'
# pickle.dump( all_fold_pred, open( pred_file, "wb" ) )
# print ' ... all predictions saved in: %s'%(pred_file)
# # ref_file = 'rnn/all_groundtruth.pkl'
# # pickle.dump( all_fold_y_test, open( ref_file, "wb" ) )
#
# # compute t-test p-values with baseline predictions
# baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl'
# baseline_preds = pickle.load(open( baseline_prediction_file, 'r' ))
#
# pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1]
# pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1]
# pvalues = (pvalue_val, pvalue_ar)
# RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0)
#
# # print(
# # 'sklearn --> valence: %.4f, arousal: %.4f\n'
# # 'Pearson Corr --> valence: %.4f, arousal: %.4f \n'
# # # % (RMSE[0], -1. , pcorr[0][0], -1)
# # % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0])
# # )
#
# s = (
# 'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n'
# % (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar)
# )
#
# print s
# log_f.write(s)
log_f.close()
# return RMSE, pcorr, pvalues
return RMSE
handlers=[
logging.FileHandler("{0}/{1}.log".format('log', args.log_filename)),
logging.StreamHandler()
]
)
logging.info("Training parameters: {}".format(vars(args)))
logging.info("Training for {} epochs.".format(args.nb_epochs))
losses = []
scores = []
scores_tr = []
t1 = time.time()
logging.info("**Evaluating initial model...**")
with torch.no_grad():
utils.evaluate(model, dl_ev, args.nb_classes)
for e in range(1, args.nb_epochs + 1):
scheduler.step()
time_per_epoch_1 = time.time()
losses_per_epoch = []
for x, y in dl_tr:
opt.zero_grad()
m = model(x.cuda())
loss = criterion(m, y.cuda())
loss.backward()
losses_per_epoch.append(loss.data.cpu().numpy())
opt.step()
time_per_epoch_2 = time.time()
losses.append(np.mean(losses_per_epoch[-20:]))
logging.info(
def generate(listing):
problem = from_file(["./solomon/" + name])[0]
for ind in hall:
tmp = evaluate(problem, ind)
yield tmp[1], tmp[0]
ScorerInvLength(),
ScorerWordNet(),
ScorerSWFreqs(),
ScorerCorpusComplexity(),
ScorerContextSimilarity(),
]
def get_features(tasks):
features = []
for (sentence, idx), subs in tasks:
for sub in subs:
features.append(map(lambda scorer: scorer.score((sentence, idx), sub), scorers))
return features
print 'initialization done'
coefs = [1,1,1,1,1]
test_dir = '../data/test-data/'
test_tasks = utils.parse_input_file(test_dir)
test_gold_file = test_dir + 'substitutions.gold-rankings'
test_gold_rankings = utils.parse_rankings_file(test_gold_file)
features = get_features(test_tasks)
transformer = MinMaxScaler().fit(features)
features = transformer.transform(features)
rankings = utils.recover_rankings(test_tasks, -np.dot(features, coefs))
print utils.evaluate(rankings, test_gold_rankings)
utils.output('unsupervised_rankings', rankings)
def main():
# Argument Settings
parser = argparse.ArgumentParser(
description='Image Tagging Classification from Naver Shopping Reviews')
parser.add_argument('--sess_name',
default='example',
type=str,
help='Session name that is loaded')
parser.add_argument('--checkpoint',
default='best',
type=str,
help='Checkpoint')
parser.add_argument('--batch_size',
default=256,
type=int,
help='batch size')
parser.add_argument('--num_workers',
default=16,
type=int,
help='The number of workers')
parser.add_argument('--num_epoch',
default=100,
type=int,
help='The number of epochs')
parser.add_argument('--model_name',
default='mobilenet_v2',
type=str,
help='[resnet50, rexnet, dnet1244, dnet1222]')
parser.add_argument('--weight_file', default='model.pth', type=str)
parser.add_argument('--optimizer', default='SGD', type=str)
parser.add_argument('--lr', default=1e-2, type=float)
parser.add_argument('--weight_decay', default=1e-5, type=float)
parser.add_argument('--learning_anneal', default=1.1, type=float)
parser.add_argument('--annealing_period', default=10, type=int)
parser.add_argument('--num_gpu', default=1, type=int)
parser.add_argument('--pretrain', action='store_true', default=False)
parser.add_argument('--mode', default='train', help='Mode')
parser.add_argument('--pause', default=0, type=int)
parser.add_argument('--iteration', default=0, type=str)
args = parser.parse_args()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# Model
logger.info('Build Model')
model = select_model(args.model_name, pretrain=args.pretrain, n_class=41)
total_param = sum([p.numel() for p in model.parameters()])
logger.info(f'Model size: {total_param} tensors')
load_weight(model, args.weight_file)
model = model.to(device)
nu.bind_model(model)
nsml.save('best')
if args.pause:
nsml.paused(scope=locals())
if args.num_epoch == 0:
return
# Set the dataset
logger.info('Set the dataset')
df = pd.read_csv(f'{DATASET_PATH}/train/train_label')
train_size = int(len(df) * 0.8)
trainset = TagImageDataset(data_frame=df[:train_size],
root_dir=f'{DATASET_PATH}/train/train_data',
transform=train_transform)
testset = TagImageDataset(data_frame=df[train_size:],
root_dir=f'{DATASET_PATH}/train/train_data',
transform=test_transform)
train_loader = DataLoader(dataset=trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
test_loader = DataLoader(dataset=testset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
criterion = nn.CrossEntropyLoss(reduction='mean')
optimizer = select_optimizer(model.parameters(), args.optimizer, args.lr,
args.weight_decay)
criterion = criterion.to(device)
if args.mode == 'train':
logger.info('Start to train!')
train_process(args=args,
model=model,
train_loader=train_loader,
test_loader=test_loader,
optimizer=optimizer,
criterion=criterion,
device=device)
elif args.mode == 'test':
nsml.load(args.checkpoint, session=args.sess_name)
logger.info('[NSML] Model loaded from {}'.format(args.checkpoint))
model.eval()
logger.info('Start to test!')
test_loss, test_acc, test_f1 = evaluate(model=model,
test_loader=test_loader,
device=device,
criterion=criterion)
logger.info(test_loss, test_acc, test_f1)
model.save_mappings(id_to_word, id_to_char, id_to_tag)
# Build the model
f_train, f_eval = model.build(**parameters)
# Reload previous model values
if opts.reload:
print('Reloading previous model...')
model.reload()
best_dev = -np.inf
best_test = -np.inf
dev_score = evaluate(parameters,
f_eval,
dev_sentences,
dev_data,
id_to_tag,
verbose=verbose)
test_score = evaluate(parameters,
f_eval,
test_sentences,
test_data,
id_to_tag,
verbose=verbose)
print("Score on dev: %.5f" % dev_score)
print("Score on test: %.5f" % test_score)
if dev_score > best_dev:
best_dev = dev_score
print("New best score on dev.")
print("Saving model to disk...")
model.save()
model.load_state_dict(torch.load(opts.load+'/model.mdl'))
n_epochs = 20 # number of epochs over the training set
Division = 2
accuracys = []
precisions = []
recalls = []
FB1s =[]
for epoch in xrange(n_epochs):
epoch_costs = []
print("Starting epoch %i..." % (epoch))
for i, index in enumerate(np.random.permutation(len(train_data))):
if i %(len(train_data)/Division) == 0:
# evaluate
eval_result = evaluate(model, dev_data, dictionaries)
accuracys.append(eval_result['accuracy'])
precisions.append(eval_result['precision'])
recalls.append(eval_result['recall'])
FB1s.append(eval_result['FB1'])
save_model_dictionaries('model', model, dictionaries, opts)
# Step 1. Remember that Pytorch accumulates gradients. We need to clear them out
# before each instance
model.zero_grad()
# Step 2. Get our inputs ready for the network, that is, turn them into Variables
# of word indices.
input_words = autograd.Variable(torch.LongTensor(train_data[index]['words']))
targets = autograd.Variable(torch.LongTensor(train_data[index]['tags']))
def train(args):
global img, tgt_caption
SEED_EVERYTHING()
batch_size = args.batch_size
epochs = args.epochs
device = torch.device(args.device)
train_dataloader, valid_dataloader = get_dataloaders(batch_size)
with open('config.json','r') as f:
model_config = json.load(f)
model = get_model(**model_config) #Seq2SeqModel(dropout_p=0.25, hidden_size=256,num_layers=1)
model.to(device)
# for param in model.encoder.parameters():
# param.requires_grad = False
print(model)
print(model.decoder.embedding.weight.requires_grad)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.7, patience=2, verbose=True, min_lr=1e-6, mode='max')
if args.resume_from is not None:
state = torch.load(args.resume_from)
model.load_state_dict(state['model_state_dict'])
optimizer.load_state_dict(state['optimizer_state_dict'])
scheduler.load_state_dict(state['scheduler_state_dict'])
if args.mixed_precision:
scaler = torch.cuda.amp.GradScaler()
loss_func = nn.CrossEntropyLoss(ignore_index=args.padding_idx)
best_bleu = 0
for epoch_i in range(epochs):
loss_meter = AverageMeter()
bleu_meter = AverageMeter()
pbar = tqdm(train_dataloader, total=len(train_dataloader))
model.train()
for step, batch in enumerate(pbar):
img = batch[0].to(device)
tgt_caption = batch[1].to(device)
optimizer.zero_grad()
if args.mixed_precision:
with torch.cuda.amp.autocast():
outputs = model(img, tgt_caption)
loss = loss_func(outputs.view(-1, args.padding_idx), tgt_caption[1:].view(-1))
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(optimizer)
scaler.update()
else:
outputs = model(img, tgt_caption)
loss = loss_func(outputs.view(-1, args.padding_idx), tgt_caption[1:].view(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
pred_captions = outputs.argmax(2).cpu().numpy()
true_captions = batch[1][1:].numpy()
bleu = calc_bleu_score(true_captions, pred_captions)
loss_meter.update(loss.item())
bleu_meter.update(bleu)
pbar.set_postfix({'loss':loss_meter.avg, 'bleu':bleu_meter.avg})
valid_loss, valid_bleu = evaluate(model, valid_dataloader, device, epoch_i, args.key, loss_func)
scheduler.step(valid_bleu)
if valid_bleu > best_bleu:
print('validation bleu improved from %.4f to %.4f'%(best_bleu,valid_loss))
print('saving model...')
torch.save({'model_state_dict':model.state_dict(),
'optimizer_state_dict':optimizer.state_dict(),
'scheduler_state_dict':scheduler.state_dict()}, f'saved_models/{args.key}/state.pth')
best_bleu = valid_bleu
print(f'Epoch: {epoch_i+1}/{epochs}, train loss:{loss_meter.avg:.4f}, train bleu:{bleu_meter.avg:.4f}\nvalid loss: {valid_loss:.4f}, valid bleu: {valid_bleu:.4f}')
torch.cuda.empty_cache()
best_test = -np.inf
count = 0
for epoch in range(n_epochs):
epoch_costs = []
print("Starting epoch %i..." % epoch)
go = True
for i, index in enumerate(np.random.permutation(len(train_data))):
count += 1
input = create_input(train_data[index], parameters, True, singletons)
new_cost = f_train(*input)
epoch_costs.append(new_cost)
if i % 50 == 0 and i > 0 == 0:
print("%i, cost average: %f" % (i, np.mean(epoch_costs[-50:])))
if count % freq_eval == 0:
go = False
dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data,
id_to_tag, dico_tags)
test_score = evaluate(parameters, f_eval, test_sentences,
test_data, id_to_tag, dico_tags)
print("Score on dev: %.5f" % dev_score)
print("Score on test: %.5f" % test_score)
if dev_score > best_dev:
best_dev = dev_score
print("New best score on dev.")
print("Saving model to disk...")
model.save()
if test_score > best_test:
best_test = test_score
print("New best score on test.")
if go:
dev_score = evaluate(parameters, f_eval, dev_sentences, dev_data,
id_to_tag, dico_tags)
freq_eval = 1000 # evaluate on dev every freq_eval steps
best_dev = -np.inf
best_test = -np.inf
count = 0
for epoch in xrange(n_epochs):
epoch_costs = []
print "Starting epoch %i..." % epoch
for i, index in enumerate(np.random.permutation(len(train_data))):
count += 1
input = create_input(train_data[index], parameters, True, singletons)
new_cost = f_train(*input)
epoch_costs.append(new_cost)
if i % 50 == 0 and i > 0 == 0:
print "%i, cost average: %f" % (i, np.mean(epoch_costs[-50:]))
if count % freq_eval == 0:
dev_score = evaluate(parameters, f_eval, dev_sentences,
dev_data, id_to_tag, dico_tags)
test_score = evaluate(parameters, f_eval, test_sentences,
test_data, id_to_tag, dico_tags)
print "Score on dev: %.5f" % dev_score
print "Score on test: %.5f" % test_score
if dev_score > best_dev:
best_dev = dev_score
print "New best score on dev."
print "Saving model to disk..."
model.save()
if test_score > best_test:
best_test = test_score
print "New best score on test."
print "Epoch %i done. Average cost: %f" % (epoch, np.mean(epoch_costs))
def validate(val_loader, net, criterion, optimizer, epoch, train_args, restore, visualize):
net.eval()
val_loss = AverageMeter()
inputs_all, gts_all, predictions_all = [], [], []
for vi, data in enumerate(val_loader):
inputs, gts = data
N = inputs.size(0)
inputs = Variable(inputs, volatile=True).cuda()
gts = Variable(gts, volatile=True).cuda()
outputs = net(inputs)
predictions = outputs.data.max(1)[1].squeeze_(1).squeeze_(0).cpu().numpy()
val_loss.update(criterion(outputs, gts).data[0] / N, N)
if random.random() > train_args['val_img_sample_rate']:
inputs_all.append(None)
else:
inputs_all.append(inputs.data.squeeze_(0).cpu())
gts_all.append(gts.data.squeeze_(0).cpu().numpy())
predictions_all.append(predictions)
acc, acc_cls, mean_iu, fwavacc = evaluate(predictions_all, gts_all, voc.num_classes)
if mean_iu > train_args['best_record']['mean_iu']:
train_args['best_record']['val_loss'] = val_loss.avg
train_args['best_record']['epoch'] = epoch
train_args['best_record']['acc'] = acc
train_args['best_record']['acc_cls'] = acc_cls
train_args['best_record']['mean_iu'] = mean_iu
train_args['best_record']['fwavacc'] = fwavacc
snapshot_name = 'epoch_%d_loss_%.5f_acc_%.5f_acc-cls_%.5f_mean-iu_%.5f_fwavacc_%.5f_lr_%.10f' % (
epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc, optimizer.param_groups[1]['lr']
)
torch.save(net.state_dict(), os.path.join(ckpt_path, exp_name, snapshot_name + '.pth'))
torch.save(optimizer.state_dict(), os.path.join(ckpt_path, exp_name, 'opt_' + snapshot_name + '.pth'))
if train_args['val_save_to_img_file']:
to_save_dir = os.path.join(ckpt_path, exp_name, str(epoch))
check_mkdir(to_save_dir)
val_visual = []
for idx, data in enumerate(zip(inputs_all, gts_all, predictions_all)):
if data[0] is None:
continue
input_pil = restore(data[0])
gt_pil = voc.colorize_mask(data[1])
predictions_pil = voc.colorize_mask(data[2])
if train_args['val_save_to_img_file']:
input_pil.save(os.path.join(to_save_dir, '%d_input.png' % idx))
predictions_pil.save(os.path.join(to_save_dir, '%d_prediction.png' % idx))
gt_pil.save(os.path.join(to_save_dir, '%d_gt.png' % idx))
val_visual.extend([visualize(input_pil.convert('RGB')), visualize(gt_pil.convert('RGB')),
visualize(predictions_pil.convert('RGB'))])
val_visual = torch.stack(val_visual, 0)
val_visual = vutils.make_grid(val_visual, nrow=3, padding=5)
writer.add_image(snapshot_name, val_visual)
print('--------------------------------------------------------------------')
print('[epoch %d], [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f]' % (
epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc))
print('best record: [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f], [epoch %d]' % (
train_args['best_record']['val_loss'], train_args['best_record']['acc'], train_args['best_record']['acc_cls'],
train_args['best_record']['mean_iu'], train_args['best_record']['fwavacc'], train_args['best_record']['epoch']))
print('--------------------------------------------------------------------')
writer.add_scalar('val_loss', val_loss.avg, epoch)
writer.add_scalar('acc', acc, epoch)
writer.add_scalar('acc_cls', acc_cls, epoch)
writer.add_scalar('mean_iu', mean_iu, epoch)
writer.add_scalar('fwavacc', fwavacc, epoch)
writer.add_scalar('lr', optimizer.param_groups[1]['lr'], epoch)
net.train()
return val_loss.avg
def run_latency_changing_topo(train_loader,
test_loader,
num_rounds,
epochs,
num_clients,
latency_nodes,
net='net',
optimizer='sgd',
lr=0.1):
"""
Runs a decentralized optimization algorithm for the given learning rate for a
number of rounds, over some network. Some nodes send their weights with a one-rounds
latency, for the entire execution. The network topology evolves over time. Outputs
the accuracies and returns them.
Params:
train_loader (array): the list of all train datasets, one per client
test_loader (array): the list of test datasets, one per client
comm_matrix (numpy.array): the communication matric modeling the network
num_rounds (int): the number of data exchanges between nodes
epochs (int): the number of optimization steps between each communication (minimum 1)
num_clients (int): the number of clients in the network
latency_nodes (array): the list of delayed nodes
net (string): the neural network framework we use
optimizer (string): the chosen optimizer, SGD by default
lr (double): the learning rate for the optimizaion algorithm
Returns:
global_model (nn.Module): the final global neural network averaging all the clients
client_models (array of Net): the list of all the final client neural networks
accs (array): the corresponding accuracies, with the same shape as lrs
"""
accs = []
global_model, client_models = model_init(num_clients, net)
opt = optimizer_init(client_models, lr, optimizer)
topos = ['centralized', 'ring', 'grid']
topo = np.random.choice(topos)
comm_matrix = create_mixing_matrix(topo, num_clients)
loss, test_loss, acc = 0.0, 0.0, 0.0
for r in range(num_rounds):
old_client_models = client_models
old_topo = topo
old_comm_matrix = comm_matrix
topo = np.random.choice(topos)
# client update
for i in range(num_clients):
loss += client_update(client_models[i],
opt[i],
train_loader[i],
epoch=epochs)
# diffuse params
diffuse_params_latency(client_models, comm_matrix, latency_nodes)
if (r > 0):
diffuse_params_latency(
old_client_models, old_comm_matrix,
np.setdiff1d(np.array(range(num_clients)), latency_nodes))
print("old topo: {}, new topo: {}".format(old_topo, topo))
average_models(global_model, client_models)
test_loss, acc = evaluate(global_model, test_loader)
print('%d-th round' % r)
print('average train loss %0.3g | test loss %0.3g | test acc: %0.3f' %
(loss / num_clients, test_loss, acc))
accs.append(acc)
return global_model, client_models, accs
(p_e,g_e, p_a, g_a) = test_perceptron_joint(FV, L_t, L_a, valid,
mode = 'Joint_constrained',
subsample = False)
#save results
savedata = (L_t, L_a, misc_t ,misc_a)
with open('Joint_perceptron_.data', 'wb') as f:
cPickle.dump(savedata, f)
with open('Joint_perceptron_.data', 'rb') as f:
(L_t2, L_a2, misc_t2 ,misc_a2) = cPickle.load(f)
#evaluate trigger predictions
errors_e = [1 for y1,y2 in zip(p_e, g_e) if y1!=y2]
validation_error_e = len(errors_e)/float(len(g_e))
print 'Trigger prediction error (accuracy)',(validation_error_e)
utils.evaluate(g_e, p_e, FV, mode = 'Trigger')
#evaluate argument predictions
pp_a = [i for sublist in p_a for i in sublist]
gg_a = [i for sublist in g_a for i in sublist]
errors_a = [1 for y1,y2 in zip(pp_a, gg_a) if y1!=y2]
validation_error_a = len(errors_a)/float(len(gg_a))
print 'Argument prediction error (accuracy)',(validation_error_a)
utils.evaluate(gg_a, pp_a, FV, mode = 'Arguments')
#save predictions
with open('Joint_perceptron_predictions_e_.data', 'wb') as f:
cPickle.dump((p_e,g_e), f)
with open('Joint_perceptron_predictions_a_.data', 'wb') as f:
cPickle.dump((p_a,g_a), f)
with open('Joint_perceptron_predictions_e_.data', 'rb') as f:
def main(argv=None):
# Choose a device and move everything there
print("Running on {}".format(device))
parser = argparse.ArgumentParser(
description="Train a transformer for a copy task")
add_optimizer_arguments(parser)
add_transformer_arguments(parser)
add_auxiliary_arguments(parser)
args = parser.parse_args(argv)
print("args:\n-----\n", args)
# Make the dataset and the model
for model_type in ['rnn', 'lstm', 'transformer']:
for max_depth in range(1, 12):
ii = 0
while ii < 10:
print(max_depth, ii)
# skip existing models
if os.path.isfile(
"/content/drive/My Drive/final_project_material/model_storage/model_"
+ model_type + "_depth_" + str(max_depth) + "_num_" +
str(ii)):
ii += 1
continue
train_set = CountTaskWithEOS(2 * max_depth + 1,
max_depth=max_depth)
test_set = CountTaskWithEOS(2 * max_depth + 1,
max_depth=max_depth)
train_loader = DataLoader(train_set,
batch_size=batch_size,
pin_memory=device == "cuda")
test_loader = DataLoader(test_set,
batch_size=batch_size,
pin_memory=device == "cuda")
if model_type == "transformer":
d_model = 8
model = SequencePredictorRecurrentTransformer(
d_model=d_model,
n_classes=3,
sequence_length=128,
attention_type="causal-linear",
n_layers=1,
n_heads=args.n_heads,
d_query=d_model, # used to be d_query
dropout=args.dropout,
softmax_temp=None,
attention_dropout=args.attention_dropout,
)
else:
d_model = 3
model = SequencePredictorRNN(d_model=d_model,
n_classes=args.n_classes,
n_layers=args.n_layers,
dropout=args.dropout,
rnn_type=model_type)
print(f"Created model:\n{model}")
model.to(device)
# Start training
optimizer = get_optimizer(model.parameters(), args)
start_epoch = 1
if args.continue_from:
start_epoch = load_model(args.continue_from, model,
optimizer, device)
lr_schedule = torch.optim.lr_scheduler.LambdaLR(
optimizer, lambda e: 1. if e < args.reduce_lr_at else 0.1)
for e in range(start_epoch, args.epochs + 1):
print('Epoch:', e)
print('Training...')
train(model, optimizer, train_loader, device)
print('Evaluating...')
acc = evaluate(model,
test_loader,
device,
return_accuracy=True)
lr_schedule.step()
if e == 40:
break
if acc >= 1:
save_model(
"/content/drive/My Drive/final_project_material/model_storage/model_"
+ model_type + "_depth_" + str(max_depth) +
"_num_" + str(ii), model, optimizer, e)
ii += 1
break
def rnn_cv( output_model_dir, model_name, pred_file, data, n_hidden=10, n_epochs=50, lr=0.001, lrd = 0.999, reg_coef= 0.01, doSmoothing=False):
doSaveModel = True
MODELDIR = output_model_dir
LOGDIR = MODELDIR
print '... model output dir: %s'%(MODELDIR)
# smooth prediction params
taille = 12
wts = np.ones(taille-1)*1./taille
wts = np.hstack((np.array([1./(2*taille)]), wts, np.array([1./(2*taille)])))
delay = (wts.shape[0]-1) / 2
print '... doSmoothing: %s'%(doSmoothing)
# # initialize global logger variable
# print '... initializing global logger variable'
# logger = logging.getLogger(__name__)
# withFile = False
# logger = settings.init(MODELDIR + 'train.log', withFile)
# perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef)
perf_file_name = LOGDIR + 'performance.log'
log_f = open(perf_file_name, 'w')
all_fold_pred = list()
all_fold_y_test = list()
all_fold_pred_smooth = list()
t0 = time.time()
X_train = data['train']['X']
y_train = data['train']['y']
id_train = data['train']['song_id']
X_test = X_train
y_test = y_train
id_test = id_train
print '... training and testing on the same data ...'
print X_train.shape, y_train.shape, X_test.shape, y_test.shape
nb_seq_train, nb_frames_train, nb_features_train = X_train.shape
nb_seq_test, nb_frames_test, nb_features_test = X_test.shape
assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST'
assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST'
dim_ouput_train = y_train.shape[2]
dim_ouput_test = y_test.shape[2]
assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST'
n_in = nb_features_train
n_out = dim_ouput_train
n_steps = nb_frames_train
validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs
model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out,
learning_rate=lr, learning_rate_decay=lrd,
L1_reg=reg_coef, L2_reg=reg_coef,
n_epochs=n_epochs, activation='tanh')
model.fit(X_train, y_train, validation_frequency=validation_frequency)
if doSaveModel:
# model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef)
# model_name = MODELDIR + 'model_baseline_predictions_as_features_431songs_normed.pkl'
model_name = MODELDIR + model_name
model.save(fpath=model_name)
pred = list()
pred_smooth = list()
for ind_seq_test in xrange(nb_seq_test):
pred.append(model.predict(X_test[ind_seq_test]))
if doSmoothing:
for ind_seq_test in xrange(nb_seq_test):
y_hat = np.array(model.predict(X_test[ind_seq_test]), dtype=float)
y_hat_smooth = np.zeros_like(y_hat, dtype=float)
y_hat_smooth[:, 0] = np.convolve(y_hat[:, 0], wts, mode='same')
y_hat_smooth[:delay, 0] = y_hat[:delay, 0]
y_hat_smooth[-delay:, 0] = y_hat[-delay:, 0]
y_hat_smooth[:, 1] = np.convolve(y_hat[:, 1], wts, mode='same')
y_hat_smooth[:delay, 1] = y_hat[:delay, 1]
y_hat_smooth[-delay:, 1] = y_hat[-delay:, 1]
pred_smooth.append(y_hat_smooth)
y_hat = np.array(pred, dtype=float)
y_hat_smooth = np.array(pred_smooth, dtype=float)
# save predictions as 3d tensors
# pred_file = LOGDIR + 'predictions_train_set_baseline_predictions_as_features_431songs_normed.pkl'
if doSmoothing:
pickle.dump( y_hat_smooth, open( pred_file, "wb" ) )
else:
pickle.dump( y_hat, open( pred_file, "wb" ) )
print ' ... predictions saved in: %s'%(pred_file)
y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2]))
y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2]))
print y_hat.shape, y_test_concat.shape
assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!'
all_fold_pred.append(y_hat.tolist())
all_fold_y_test.append(y_test_concat.tolist())
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0])
s = (
'training_data valence: %.4f %.4f arousal: %.4f %.4f\n'
% (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])
)
print s
log_f.write(s)
if doSmoothing:
y_hat_smooth = np.reshape(y_hat_smooth, (y_hat_smooth.shape[0]*y_hat_smooth.shape[1], y_hat_smooth.shape[2]))
# all_fold_pred_smooth.append(y_hat_smooth.tolist())
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat_smooth, id_test.shape[0])
s = (
'training_data valence: %.4f %.4f arousal: %.4f %.4f\n'
% (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])
)
print s
log_f.write(s)
EMO = 'valence'
doPlot = False
if doPlot:
fig, ax = plt.subplots()
x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0])
if EMO == 'valence':
ax.plot(x1, y_test_concat[:, 0], 'o', label="Data")
# ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction")
ax.plot(x1, y_hat[:,0], 'ro', label="OLS prediction")
else:
ax.plot(x1, y_test_concat[:, 1], 'o', label="Data")
ax.plot(x1, y_hat[:,1], 'ro', label="OLS prediction")
plt.title(EMO + ' on Train subset')
ax.legend(loc="best")
plt.show()
# plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png')
doPlotTrain = False
if doPlotTrain:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_train[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_train[0])
guess = model.predict(X_train[0])
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
doPlotTest = False
if doPlotTest:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_test[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_test[0])
# guess = model.predict(X_test[0])
guess = y_hat[0]
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
print "... Elapsed time: %f" % (time.time() - t0)
all_fold_pred = [item for sublist in all_fold_pred for item in sublist]
all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist]
all_fold_pred = np.array(all_fold_pred, dtype=float)
all_fold_y_test = np.array(all_fold_y_test, dtype=float)
print all_fold_pred.shape, all_fold_y_test.shape
return RMSE, pcorr
with open('models/binary/' + model_name + '_config.json', 'w') as f:
f.write(binary_model.config.to_json_string())
if do_evaluate:
print("loading model...")
config = BertConfig('models/binary/' + model_name + '_config.json')
eval_model = BertForSequenceClassification(config, num_labels = num_labels)
eval_model.load_state_dict(torch.load("models/binary/" + model_name + ".pt"))
eval_model.cuda()
eval_model.eval()
print("loading data...")
# Note these are different, see indices
pos_dataloader = processor.get_dataloader(DATA_DIR, "neg_dev_matched", tokenizer, batch_size = 1, a_idx = 6, b_idx = 7)
neg_dataloader = processor.get_dataloader(DATA_DIR, "neg_dev_matched", tokenizer, batch_size = 1, a_idx = 8, b_idx = 7)
evaluate(eval_model, pos_dataloader, neg_dataloader, "experiments/may12_3/binary_finetune", 3, DEBUG=True)
if other:
config = BertConfig('models/binary/' + model_name + '_config.json')
eval_model = BertForSequenceClassification(config, num_labels = num_labels)
eval_model.load_state_dict(torch.load("models/binary/" + model_name + ".pt"))
eval_model.to('cuda:0')
eval_model.eval()
eval_pos_dataloader = processor.get_dataloader(DATA_DIR, "binary_dev_mismatched", tokenizer, batch_size = 10, a_idx = 6, b_idx = 7, label_idx = 5)
#eval_neg_dataloader = processor.get_dataloader(DATA_DIR, "neg_binary_dev_mismatched", tokenizer, batch_size = 10, a_idx = 8, b_idx = 7, label_idx = 5)
simple_evaluate(eval_model, eval_pos_dataloader, "experiments/binary_" + model_name + "_pos_preds")
#simple_evaluate(eval_model, eval_neg_dataloader, "experiments/binary_finetune_neg_preds")
# here is your result file which will be evaluated
result_file = "../data/result.txt"
# test set you separate from dataset
test_file = "../data/test.txt"
if __name__ == "__main__":
if len(sys.argv) == 3:
result_file = sys.argv[1]
test_file = sys.argv[2]
elif len(sys.argv) != 1:
print("Usage: %s <result file> <test file>\n" % sys.argv[0])
exit(0)
result_graph = Graph(result_file)
test_graph = Graph(test_file)
result_graph.read()
test_graph.read()
result_set = set()
test_set = set()
result_edge = result_graph.all_edges()
for a, link in result_edge.items():
for b in link:
result_set.add((a, b))
test_edge = test_graph.all_edges()
for a, link in test_edge.items():
for b in link:
test_set.add((a, b))
evaluate(test_set, result_set)
# for i, data in enumerate(train_data):
# input = create_input(data, parameters, True, use_gaze, pos, singletons)
# train_cost.append(f_plot_cost(*input))
# dev_cost = []
# for i, data in enumerate(dev_data):
# input = create_input(data, parameters, True, use_gaze, pos, singletons)
# dev_cost.append(f_plot_cost(*input))
# test_cost = []
# for i, data in enumerate(test_data):
# input = create_input(data, parameters, True, use_gaze, pos, singletons)
# test_cost.append(f_plot_cost(*input))
print "dev:"
dev_score = evaluate(parameters, f_eval, dev_sentences,
dev_data, id_to_tag, tag_to_id, use_gaze, pos, post_process, list_prefix, label_cnt)
print "test:"
test_score = evaluate(parameters, f_eval, test_sentences,
test_data, id_to_tag, tag_to_id, use_gaze, pos, post_process, list_prefix, label_cnt)
fw.write('\t\t'.join([str(epoch), str(dev_score), str(test_score)]) + '\n')
# cost_str = '\t\t'.join(
# [str(epoch), str(np.mean(train_cost)), str(np.mean(dev_cost)), str(np.mean(test_cost)), str(dev_score), str(test_score)])
# fw.write(cost_str + '\n')
#print "Score on train: %.5f" % train_score
print "Score on dev: %.5f" % dev_score
print "Score on test: %.5f" % test_score
if dev_score > best_dev:
best_dev = dev_score
print "New best score on dev."
model, params = models.create_model(m, n_inputs, n_outputs)
if torch.cuda.device_count() > 0:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model.cuda()
model = utils.load_net_weights(model, best_model)
model.eval()
dataset_loader = torch.utils.data.DataLoader(
dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.n_threads,
worker_init_fn=lambda x: utils.worker_init_fn(x))
temp_results = utils.evaluate(dataset,
dataset_loader,
model,
save_prediction=args.save_prediction,
model_name=model_info['name'])
results = pd.concat([results, temp_results])
#-------------------------------------------------------------------------------
# Save evaluation
print(results.head())
try:
os.mkdir('./metrics/')
except:
pass
if args.name == '':
name = './metrics/{}_{}_{}_{}x{}x{}_{}.csv'.format(
model_info['name'], args.batch_size, args.n_subvolumes, args.sv_d,
args.sv_h, args.sv_w, args.seed)
else:
Division = 2
accuracys = []
precisions = []
recalls = []
FB1s = []
for epoch in xrange(
n_epochs
): # again, normally you would NOT do 300 epochs, it is toy data
epoch_costs = []
print("Starting epoch %i..." % (epoch))
for i, index in enumerate(np.random.permutation(len(train_data))):
if i % (len(train_data) / Division) == 0:
# evaluate
eval_result = evaluate(model, dev_data, dictionaries, opts.lower)
accuracys.append(eval_result['accuracy'])
precisions.append(eval_result['precision'])
recalls.append(eval_result['recall'])
FB1s.append(eval_result['FB1'])
# Step 1. Remember that Pytorch accumulates gradients. We need to clear them out
# before each instance
model.zero_grad()
# Step 2. Get our inputs ready for the network, that is, turn them into Variables
# of word indices.
input_words = autograd.Variable(
torch.LongTensor(train_data[index]['words']))
targets = autograd.Variable(torch.LongTensor(
train_data[index]['tags']))
best_dev = -np.inf
best_test = -np.inf
count = 0
for epoch in xrange(n_epochs):
epoch_costs = []
print "Starting epoch %i..." % epoch
for i, index in enumerate(np.random.permutation(len(train_data))):
count += 1
input = create_input(train_data[index], parameters, True, singletons)
new_cost = f_train(*input)
epoch_costs.append(new_cost)
if i % 50 == 0 and i > 0 == 0:
print "%i, cost average: %f" % (i, np.mean(epoch_costs[-50:]))
if count % freq_eval == 0:
dev_score, dev_eval_lines_prob = evaluate(parameters, f_eval,
dev_sentences, dev_data,
id_to_tag, dico_tags)
test_score, test_eval_lines_prob = evaluate(
parameters, f_eval, test_sentences, test_data, id_to_tag,
dico_tags)
print "Score on dev: %.5f" % dev_score
print "Score on test: %.5f" % test_score
if dev_score > best_dev:
best_dev = dev_score
best_dev_eval_lines = dev_eval_lines_prob
test_with_best_dev = test_score
print "New best score on dev."
print "Saving model to disk..."
model.save()
if test_score > best_test:
best_test = test_score
def rnn_cv(output_model_dir,
model_name,
pred_file,
data,
n_hidden=10,
n_epochs=50,
lr=0.001,
lrd=0.999,
reg_coef=0.01,
doSmoothing=False):
doSaveModel = True
MODELDIR = output_model_dir
LOGDIR = MODELDIR
print '... model output dir: %s' % (MODELDIR)
# smooth prediction params
taille = 12
wts = np.ones(taille - 1) * 1. / taille
wts = np.hstack(
(np.array([1. / (2 * taille)]), wts, np.array([1. / (2 * taille)])))
delay = (wts.shape[0] - 1) / 2
print '... doSmoothing: %s' % (doSmoothing)
# # initialize global logger variable
# print '... initializing global logger variable'
# logger = logging.getLogger(__name__)
# withFile = False
# logger = settings.init(MODELDIR + 'train.log', withFile)
# perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef)
perf_file_name = LOGDIR + 'performance.log'
log_f = open(perf_file_name, 'w')
all_fold_pred = list()
all_fold_y_test = list()
all_fold_pred_smooth = list()
t0 = time.time()
X_train = data['train']['X']
y_train = data['train']['y']
id_train = data['train']['song_id']
X_test = X_train
y_test = y_train
id_test = id_train
print '... training and testing on the same data ...'
print X_train.shape, y_train.shape, X_test.shape, y_test.shape
nb_seq_train, nb_frames_train, nb_features_train = X_train.shape
nb_seq_test, nb_frames_test, nb_features_test = X_test.shape
assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST'
assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST'
dim_ouput_train = y_train.shape[2]
dim_ouput_test = y_test.shape[2]
assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST'
n_in = nb_features_train
n_out = dim_ouput_train
n_steps = nb_frames_train
validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs
model = rnn_model.MetaRNN(n_in=n_in,
n_hidden=n_hidden,
n_out=n_out,
learning_rate=lr,
learning_rate_decay=lrd,
L1_reg=reg_coef,
L2_reg=reg_coef,
n_epochs=n_epochs,
activation='tanh')
model.fit(X_train, y_train, validation_frequency=validation_frequency)
if doSaveModel:
# model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef)
# model_name = MODELDIR + 'model_baseline_predictions_as_features_431songs_normed.pkl'
model_name = MODELDIR + model_name
model.save(fpath=model_name)
pred = list()
pred_smooth = list()
for ind_seq_test in xrange(nb_seq_test):
pred.append(model.predict(X_test[ind_seq_test]))
if doSmoothing:
for ind_seq_test in xrange(nb_seq_test):
y_hat = np.array(model.predict(X_test[ind_seq_test]), dtype=float)
y_hat_smooth = np.zeros_like(y_hat, dtype=float)
y_hat_smooth[:, 0] = np.convolve(y_hat[:, 0], wts, mode='same')
y_hat_smooth[:delay, 0] = y_hat[:delay, 0]
y_hat_smooth[-delay:, 0] = y_hat[-delay:, 0]
y_hat_smooth[:, 1] = np.convolve(y_hat[:, 1], wts, mode='same')
y_hat_smooth[:delay, 1] = y_hat[:delay, 1]
y_hat_smooth[-delay:, 1] = y_hat[-delay:, 1]
pred_smooth.append(y_hat_smooth)
y_hat = np.array(pred, dtype=float)
y_hat_smooth = np.array(pred_smooth, dtype=float)
# save predictions as 3d tensors
# pred_file = LOGDIR + 'predictions_train_set_baseline_predictions_as_features_431songs_normed.pkl'
if doSmoothing:
pickle.dump(y_hat_smooth, open(pred_file, "wb"))
else:
pickle.dump(y_hat, open(pred_file, "wb"))
print ' ... predictions saved in: %s' % (pred_file)
y_hat = np.reshape(y_hat,
(y_hat.shape[0] * y_hat.shape[1], y_hat.shape[2]))
y_test_concat = np.reshape(
y_test, (y_test.shape[0] * y_test.shape[1], y_test.shape[2]))
print y_hat.shape, y_test_concat.shape
assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!'
all_fold_pred.append(y_hat.tolist())
all_fold_y_test.append(y_test_concat.tolist())
RMSE, pcorr, error_per_song, mean_per_song = evaluate(
y_test_concat, y_hat, id_test.shape[0])
s = ('training_data valence: %.4f %.4f arousal: %.4f %.4f\n' %
(RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]))
print s
log_f.write(s)
if doSmoothing:
y_hat_smooth = np.reshape(
y_hat_smooth, (y_hat_smooth.shape[0] * y_hat_smooth.shape[1],
y_hat_smooth.shape[2]))
# all_fold_pred_smooth.append(y_hat_smooth.tolist())
RMSE, pcorr, error_per_song, mean_per_song = evaluate(
y_test_concat, y_hat_smooth, id_test.shape[0])
s = ('training_data valence: %.4f %.4f arousal: %.4f %.4f\n' %
(RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0]))
print s
log_f.write(s)
EMO = 'valence'
doPlot = False
if doPlot:
fig, ax = plt.subplots()
x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0])
if EMO == 'valence':
ax.plot(x1, y_test_concat[:, 0], 'o', label="Data")
# ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction")
ax.plot(x1, y_hat[:, 0], 'ro', label="OLS prediction")
else:
ax.plot(x1, y_test_concat[:, 1], 'o', label="Data")
ax.plot(x1, y_hat[:, 1], 'ro', label="OLS prediction")
plt.title(EMO + ' on Train subset')
ax.legend(loc="best")
plt.show()
# plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png')
doPlotTrain = False
if doPlotTrain:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_train[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_train[0])
guess = model.predict(X_train[0])
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
doPlotTest = False
if doPlotTest:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_test[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_test[0])
# guess = model.predict(X_test[0])
guess = y_hat[0]
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
print "... Elapsed time: %f" % (time.time() - t0)
all_fold_pred = [item for sublist in all_fold_pred for item in sublist]
all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist]
all_fold_pred = np.array(all_fold_pred, dtype=float)
all_fold_y_test = np.array(all_fold_y_test, dtype=float)
print all_fold_pred.shape, all_fold_y_test.shape
return RMSE, pcorr
input_zip = ZipFile(input_zip)
return {name: input_zip.read(name) for name in input_zip.namelist()}
data = extract_zip('/media/dsshim/nyu_v2/nyu_test.zip')
from io import BytesIO
rgb = np.load(BytesIO(data['eigen_test_rgb.npy']))
depth = np.load(BytesIO(data['eigen_test_depth.npy']))
crop = np.load(BytesIO(data['eigen_test_crop.npy']))
print('Test data loaded.\n')
if args.model_type == 'densedepth':
model = Model()
else:
model = ResNet(layers=args.layers)
model.load_state_dict(
torch.load('checkpoints/%s_%d.pth' % (args.model_type, args.layers)))
model = model.cuda()
model.eval()
start = time.time()
print('Testing...')
e = evaluate(model, rgb, depth, crop, batch_size=1)
end = time.time()
print('\nTest time', end - start, 's')
def run(om,options,i):
if options.multiling:
outdir = options.outdir
else:
cur_treebank = om.languages[i]
outdir = cur_treebank.outdir
if options.shared_task:
outdir = options.shared_task_outdir
if not options.predict: # training
fineTune = False
start_from = 1
if options.continueModel is None:
continueTraining = False
else:
continueTraining = True
trainedModel = options.continueModel
if options.fineTune:
fineTune = True
else:
start_from = options.first_epoch - 1
if not continueTraining:
print 'Preparing vocab'
if options.multiling:
path_is_dir=True,
words, w2i, pos, cpos, rels, langs, ch = utils.vocab(om.languages,\
path_is_dir,
options.shareWordLookup,\
options.shareCharLookup)
else:
words, w2i, pos, cpos, rels, langs, ch = utils.vocab(cur_treebank.trainfile)
paramsfile = os.path.join(outdir, options.params)
with open(paramsfile, 'w') as paramsfp:
print 'Saving params to ' + paramsfile
pickle.dump((words, w2i, pos, rels, cpos, langs,
options, ch), paramsfp)
print 'Finished collecting vocab'
else:
paramsfile = os.path.join(outdir, options.params)
with open(paramsfile, 'rb') as paramsfp:
print 'Load params from ' + paramsfile
words, w2i, pos, rels, cpos, langs, options, ch = pickle.load(paramsfp)
print 'Finished loading vocab'
max_epochs = options.first_epoch + options.epochs
print 'Initializing blstm arc hybrid:'
parser = ArcHybridLSTM(words, pos, rels, cpos, langs, w2i,
ch, options)
if continueTraining:
if not fineTune:
# continue training only, not doing fine tuning
options.first_epoch = start_from + 1
max_epochs = options.epochs
else:
# fine tune model
options.first_epoch = options.epochs + 1
max_epochs = options.first_epoch + 15
print 'Fine tune model for another', max_epochs - options.first_epoch, 'epochs'
parser.Load(trainedModel)
best_multi_las = -1
best_multi_epoch = 0
if continueTraining:
train_stats = codecs.open(os.path.join(outdir, 'train.stats'), 'a', encoding='utf-8')
else:
train_stats = codecs.open(os.path.join(outdir, 'train.stats'), 'w', encoding='utf-8')
for epoch in xrange(options.first_epoch, max_epochs + 1):
print 'Starting epoch ' + str(epoch)
if options.multiling:
traindata = list(utils.read_conll_dir(om.languages, "train", options.max_sentences))
else:
traindata = list(utils.read_conll(cur_treebank.trainfile, cur_treebank.iso_id,options.max_sentences))
parser.Train(traindata)
train_stats.write(unicode('Epoch ' + str(epoch) + '\n'))
print 'Finished epoch ' + str(epoch)
model_file = os.path.join(outdir, options.model + '.tmp')
parser.Save(model_file)
if options.pred_dev: # use the model to predict on dev data
if options.multiling:
pred_langs = [lang for lang in om.languages if lang.pred_dev] # languages which have dev data on which to predict
for lang in pred_langs:
lang.outfilename = os.path.join(lang.outdir, 'dev_epoch_' + str(epoch) + '.conllu')
print "Predicting on dev data for " + lang.name
devdata = utils.read_conll_dir(pred_langs,"dev")
pred = list(parser.Predict(devdata))
if len(pred)>0:
utils.write_conll_multiling(pred,pred_langs)
else:
print "Warning: prediction empty"
if options.pred_eval:
total_las = 0
for lang in pred_langs:
print "Evaluating dev prediction for " + lang.name
las_score = utils.evaluate(lang.dev_gold, lang.outfilename,om.conllu)
total_las += las_score
train_stats.write(unicode('Dev LAS ' + lang.name + ': ' + str(las_score) + '\n'))
if options.model_selection:
if total_las > best_multi_las:
best_multi_las = total_las
best_multi_epoch = epoch
else: # monolingual case
if cur_treebank.pred_dev:
print "Predicting on dev data for " + cur_treebank.name
devdata = utils.read_conll(cur_treebank.devfile, cur_treebank.iso_id)
cur_treebank.outfilename = os.path.join(outdir, 'dev_epoch_' + str(epoch) + ('.conll' if not om.conllu else '.conllu'))
pred = list(parser.Predict(devdata))
utils.write_conll(cur_treebank.outfilename, pred)
if options.pred_eval:
print "Evaluating dev prediction for " + cur_treebank.name
las_score = utils.evaluate(cur_treebank.dev_gold, cur_treebank.outfilename, om.conllu)
if options.model_selection:
if las_score > cur_treebank.dev_best[1]:
cur_treebank.dev_best = [epoch, las_score]
train_stats.write(unicode('Dev LAS ' + cur_treebank.name + ': ' + str(las_score) + '\n'))
if epoch == max_epochs: # at the last epoch choose which model to copy to barchybrid.model
if not options.model_selection:
best_epoch = options.epochs # take the final epoch if model selection off completely (for example multilingual case)
else:
if options.multiling:
best_epoch = best_multi_epoch
else:
best_epoch = cur_treebank.dev_best[0] # will be final epoch by default if model selection not on for this treebank
if cur_treebank.model_selection:
print "Best dev score of " + str(cur_treebank.dev_best[1]) + " found at epoch " + str(cur_treebank.dev_best[0])
bestmodel_file = os.path.join(outdir,"barchybrid.model.tmp")
model_file = os.path.join(outdir,"barchybrid.model")
if fineTune:
model_file = os.path.join(outdir,"barchybrid.tuned.model")
print "Best epoch: " + str(best_epoch)
print "Copying " + bestmodel_file + " to " + model_file
copyfile(bestmodel_file,model_file)
train_stats.close()
else: #if predict - so
# import pdb;pdb.set_trace()
eval_type = options.evaltype
print "Eval type: ", eval_type
if eval_type == "train":
if options.multiling:
for l in om.languages:
l.test_gold = l.test_gold.replace('test', 'train')
else:
cur_treebank.testfile = cur_treebank.trainfile
cur_treebank.test_gold = cur_treebank.trainfile
elif eval_type == "dev":
if options.multiling:
for l in om.languages:
l.test_gold = l.test_gold.replace('test', 'dev')
else:
cur_treebank.testfile = cur_treebank.devfile
cur_treebank.test_gold = cur_treebank.devfile
if options.multiling:
modeldir = options.modeldir
if options.fineTune:
prefix = [os.path.join(outdir, os.path.basename(l.test_gold) + '-tuned') for l in om.languages]
else:
prefix = [os.path.join(outdir, os.path.basename(l.test_gold)) for l in om.languages]
else:
modeldir = om.languages[i].modeldir
if options.fineTune:
prefix = os.path.join(outdir, os.path.basename(cur_treebank.testfile)) + '-tuned'
else:
prefix = os.path.join(outdir, os.path.basename(cur_treebank.testfile))
if not options.extract_vectors:
prefix = None
params = os.path.join(modeldir, options.params)
print 'Reading params from ' + params
with open(params, 'r') as paramsfp:
words, w2i, pos, rels, cpos, langs, stored_opt, ch = pickle.load(paramsfp)
parser = ArcHybridLSTM(words, pos, rels, cpos, langs, w2i,
ch, stored_opt)
if options.fineTune:
options.model = options.model.replace('.model', '.tuned.model')
model = os.path.join(modeldir, options.model)
parser.Load(model)
if options.multiling:
testdata = utils.read_conll_dir(om.languages, eval_type)
else:
testdata = utils.read_conll(cur_treebank.testfile, cur_treebank.iso_id)
ts = time.time()
if options.multiling:
for l in om.languages:
l.outfilename = os.path.join(outdir, eval_type + "-" + l.outfilename)
pred = list(parser.Predict(testdata, prefix))
utils.write_conll_multiling(pred,om.languages)
else:
if cur_treebank.outfilename:
cur_treebank.outfilename = os.path.join(outdir, eval_type + "-" + cur_treebank.outfilename)
else:
cur_treebank.outfilename = os.path.join(outdir, 'out' + ('.conll' if not om.conllu else '.conllu'))
utils.write_conll(cur_treebank.outfilename, parser.Predict(testdata, prefix))
te = time.time()
if options.pred_eval:
if options.multiling:
for l in om.languages:
print "Evaluating on " + l.name
score = utils.evaluate(l.test_gold, l.outfilename, om.conllu)
print "Obtained LAS F1 score of %.2f on %s" %(score, l.name)
else:
print "Evaluating on " + cur_treebank.name
score = utils.evaluate(cur_treebank.test_gold, cur_treebank.outfilename, om.conllu)
print "Obtained LAS F1 score of %.2f on %s" %(score,cur_treebank.name)
print 'Finished predicting'
raise AssertionError("Wrong Model Type!")
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
criterion.to(device)
best_val_acc = 0.0
train_losses, train_accuracies = [], []
valid_losses, valid_accuracies = [], []
for epoch in range(NUM_EPOCHS):
train_loss, train_accuracy = train(model, device, train_loader, criterion, optimizer, epoch)
valid_loss, valid_accuracy, valid_results = evaluate(model, device, valid_loader, criterion)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
train_accuracies.append(train_accuracy)
valid_accuracies.append(valid_accuracy)
is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric.
if is_best:
best_val_acc = valid_accuracy
torch.save(model, os.path.join(PATH_OUTPUT, save_file))
plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies)
best_model = torch.load(os.path.join(PATH_OUTPUT, save_file))
def main():
cfg = TrainConfig().parse()
print(cfg.name)
result_dir = os.path.join(
cfg.result_root,
cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
train_session = cfg.train_session
train_set = prepare_dataset(cfg.feature_root, train_session, cfg.feat,
cfg.label_root)
batch_per_epoch = len(train_set) // cfg.sess_per_batch
val_session = cfg.val_session
val_set = prepare_dataset(cfg.feature_root, val_session, cfg.feat,
cfg.label_root)
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
# load backbone model
if cfg.network == "tsn":
model = networks.ConvTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
elif cfg.network == "rtsn":
model = networks.ConvRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
# get the embedding
input_ph = tf.placeholder(tf.float32,
shape=[None, cfg.num_seg, None, None, None])
label_ph = tf.placeholder(tf.float32, shape=[None])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model.forward(input_ph, dropout_ph)
if cfg.normalized:
embedding = tf.nn.l2_normalize(model.hidden,
axis=-1,
epsilon=1e-10)
else:
embedding = model.hidden
# variable for visualizing the embeddings
emb_var = tf.Variable([0.0], name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
diffs = utils.all_diffs_tf(embedding, embedding)
all_dist = utils.cdist_tf(diffs)
tf.summary.histogram('embedding_dists', all_dist)
# whether to use softplus
if cfg.no_soft:
margin = cfg.alpha
else:
margin = "soft"
metric_loss, num_active, diff, weights, fp, cn = networks.batch_hard(
all_dist, label_ph, margin)
regularization_loss = tf.reduce_sum(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
total_loss = metric_loss + regularization_loss * cfg.lambda_l2
tf.summary.scalar('active_ratio', num_active)
tf.summary.scalar('learning_rate', lr_ph)
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, tf.global_variables())
saver = tf.train.Saver(max_to_keep=10)
summary_op = tf.summary.merge_all()
# session iterator for session sampling
feat_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
label_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
train_data = session_generator(feat_paths_ph,
label_paths_ph,
sess_per_batch=cfg.sess_per_batch,
num_threads=2,
shuffled=False,
preprocess_func=model.prepare_input)
train_sess_iterator = train_data.make_initializable_iterator()
next_train = train_sess_iterator.get_next()
# prepare validation data
val_feats = []
val_labels = []
for session in val_set:
eve_batch, lab_batch, _ = load_data_and_label(
session[0], session[1], model.prepare_input_test
) # use prepare_input_test for testing time
val_feats.append(eve_batch)
val_labels.append(lab_batch)
val_feats = np.concatenate(val_feats, axis=0)
val_labels = np.concatenate(val_labels, axis=0)
print("Shape of val_feats: ", val_feats.shape)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
for v in val_labels:
fout.write('%d\n' % int(v))
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
# load pretrain model, if needed
if cfg.model_path:
print("Restoring pretrained model: %s" % cfg.model_path)
saver.restore(sess, cfg.model_path)
################## Training loop ##################
epoch = -1
while epoch < cfg.max_epochs - 1:
step = sess.run(global_step, feed_dict=None)
epoch = step // batch_per_epoch
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.001**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# prepare data for this epoch
random.shuffle(train_set)
feat_paths = [path[0] for path in train_set]
label_paths = [path[1] for path in train_set]
# reshape a list to list of list
# interesting hacky code from: https://stackoverflow.com/questions/10124751/convert-a-flat-list-to-list-of-list-in-python
feat_paths = list(zip(*[iter(feat_paths)] *
cfg.sess_per_batch))
label_paths = list(
zip(*[iter(label_paths)] * cfg.sess_per_batch))
sess.run(train_sess_iterator.initializer,
feed_dict={
feat_paths_ph: feat_paths,
label_paths_ph: label_paths
})
# for each epoch
batch_count = 1
while True:
try:
# First, sample sessions for a batch
start_time_select = time.time()
eve, se, lab = sess.run(next_train)
select_time1 = time.time() - start_time_select
# Second, select samples for a batch
batch_idx = select_batch(lab, cfg.batch_size)
eve = eve[batch_idx]
lab = lab[batch_idx]
# Third, perform training on a batch
start_time_train = time.time()
err, _, step, summ, diff_v, weights_v, fp_v, cn_v, dist_v = sess.run(
[
total_loss, train_op, global_step, summary_op,
diff, weights, fp, cn, all_dist
],
feed_dict={
input_ph: eve,
dropout_ph: cfg.keep_prob,
label_ph: np.squeeze(lab),
lr_ph: learning_rate
})
train_time = time.time() - start_time_train
pdb.set_trace()
print ("Epoch: [%d][%d/%d]\tEvent num: %d\tSelect_time: %.3f\tTrain_time: %.3f\tLoss %.4f" % \
(epoch+1, batch_count, batch_per_epoch, eve.shape[0], select_time1, train_time, err))
summary = tf.Summary(value=[
tf.Summary.Value(tag="train_loss",
simple_value=err),
tf.Summary.Value(tag="select_time1",
simple_value=select_time1)
])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
batch_count += 1
except tf.errors.OutOfRangeError:
print("Epoch %d done!" % (epoch + 1))
break
# validation on val_set
print("Evaluating on validation set...")
val_embeddings, _ = sess.run([embedding, set_emb],
feed_dict={
input_ph: val_feats,
dropout_ph: 1.0
})
mAP, _ = utils.evaluate(val_embeddings, val_labels)
summary = tf.Summary(value=[
tf.Summary.Value(tag="Valiation mAP", simple_value=mAP)
])
summary_writer.add_summary(summary, step)
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(
result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
# save model
saver.save(sess,
os.path.join(result_dir, cfg.name + '.ckpt'),
global_step=step)
from utils import evaluate
# here is your result file which will be evaluated
result_file = "../data/result.txt"
# test set you separate from dataset
test_file = "../data/test.txt"
if __name__ == "__main__":
if len(sys.argv) == 3:
result_file = sys.argv[1]
test_file = sys.argv[2]
elif len(sys.argv) != 1:
print("Usage: %s <result file> <test file>\n" % sys.argv[0])
exit(0)
result_graph = Graph(result_file)
test_graph = Graph(test_file)
result_graph.read()
test_graph.read()
result_set = set()
test_set = set()
result_edge = result_graph.all_edges()
for a, link in result_edge.items():
for b in link:
result_set.add((a, b))
test_edge = test_graph.all_edges()
for a, link in test_edge.items():
for b in link:
test_set.add((a, b))
evaluate(test_set, result_set)
# MUTATION
if np.random.rand() < mutation:
new_agent = mutate(new_agent, bounds[0], bounds[1])
agents[x] = new_agent
iteration_history.append(new_agent)
agents_history[iteration] = np.clip(iteration_history, bounds[0],
bounds[1])
fitness_history[iteration] = [
evaluate(function, agent) for agent in agents
]
iteration += 1
if (max_iterations != None) and (max_iterations == iteration):
end = True
if (target_fitness != None) and (target_fitness == best_fitness):
end = True
# calculate total fitness for all agents
fitnesses = np.array([[evaluate(function, agent) for agent in agents]
for agents in agents_history])
best_agent_id = int(np.argmin(fitnesses) / max_iterations) % population
best_agent = agents_history[:, best_agent_id, :]
best_fitness = fitnesses[:, best_agent_id]
return best_agent, best_fitness, agents_history #best_fitness_history
if __name__ == "__main__":
best_agent, best_fitness, agents_history = differential_evolution()
print(evaluate("rosenbrock", best_agent))
tst_song = len(song_id_tst)
# Create linear regression object
regr = linear_model.LinearRegression()
# Train the model using the training sets
regr.fit(X_train, y_train)
# The coefficients
# print('Coefficients: \n', regr.coef_)
# What we predict
y_hat = regr.predict(X_test)
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test, y_hat, tst_song)
All_stack =np.hstack(( error_per_song, mean_per_song ))
print' Error per song (ar/val) Mean_per_song (ar/val) :\n'
print(All_stack)
print '\n'
print'song_id :'
print(song_id_tst)
print '\n'
#print('Error per song: \n', Error_per_song)
print(
'sklearn --> arrousal : %.4f, valence : %.4f\n'
'Pearson Corr --> arrousal : %.4f, valence : %.4f \n'
% (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0])
def genetic_algorithm(mutation=0.1,
population=10,
function="rosenbrock",
max_iterations=5000,
target_fitness=None,
dimensions=2,
selection="elitism"):
"""
:param crossover:
:param mutation:
:param selection:
:param pop_size:
:return:
- init agents x at random positions in search space
- calculate fitness value of each agent
- while(not end) : (end can be number of iterations, adequate fitness reached, ...)
- for each agent :
- calculate whether selected (selection strategy)
- for each agent :
- if selected, use as parent
- otherwise, calculate cross-over with parents based on selection and mutation
- calculate fitness value in the new iteration
- Pick agent with best fitness and return it as solution
"""
bounds = [-2.4, 2.4] if function == "rosenbrock" else [-5.12, 5.12]
agents = [[
np.random.uniform(low=bounds[0], high=bounds[1])
for _ in range(dimensions)
] for _ in range(population)]
for agent in agents:
np.clip(agent, bounds[0], bounds[1])
end = False
iteration = 1
best_fitness = None
first_run = True
agents_history = np.zeros((max_iterations, len(agents), 2), dtype=float)
fitness_history = np.zeros((max_iterations, len(agents)), dtype=float)
fitness_history[0] = [evaluate(function, agent) for agent in agents]
num_selected = int(
population /
5) if selection == "elitism" or selection == "roulette" else int(
population / 5 * 4) if selection == "steady" else int(population /
2)
while not end:
iteration_history = []
selected_agents = []
# SELECTION
if selection == "elitism" or selection == "steady":
selected_agents = np.argpartition(fitness_history[iteration - 1],
num_selected + 1)
elif selection == "tournament":
random_order = np.random.choice(range(population),
population,
replace=False)
left_bracket = random_order[:num_selected]
right_bracket = random_order[num_selected:]
for i in range(num_selected):
selected_agent = left_bracket[i] if fitness_history[
iteration - 1][left_bracket[i]] > fitness_history[
iteration - 1][right_bracket[i]] else right_bracket[
i] if fitness_history[iteration - 1][
left_bracket[i]] < fitness_history[
iteration -
1][right_bracket[i]] else np.random.choice(
[left_bracket[i], right_bracket[i]])
selected_agents.append(selected_agent)
elif selection == "roulette":
total_fitness = np.sum(fitness_history[iteration - 1])
random_order = np.random.choice(range(population),
population,
replace=False)
roulette_selection = {
int(i): fitness_history[iteration - 1][int(i)]
for i in random_order
}
chance = np.random.uniform(0, total_fitness)
i = 0
current = 0
for key, value in roulette_selection.items():
current += value
if current > chance:
selected_agents.append(key)
i += 1
if i >= num_selected:
break
else: # in case of error, default to elitism
selected_agents = np.argpartition(fitness_history[iteration - 1],
num_selected + 1)
if first_run:
first_run = False
iteration = 0
for x in range(len(agents)):
if x in selected_agents[:num_selected]:
new_agent = agents[x]
else:
# CROSSOVER
if len(selected_agents) > 0:
dad = np.random.choice(selected_agents)
mom = np.random.choice(selected_agents)
while dad == mom:
mom = np.random.choice(selected_agents)
else: # Roulette selection is special
parents = np.random.choice(range(population),
2,
replace=False)
dad = parents[0]
mom = parents[1]
new_agent = cross_over(agents[dad], agents[mom])
# MUTATION
if np.random.rand() < mutation:
new_agent = mutate(new_agent, bounds[0], bounds[1])
agents[x] = new_agent
iteration_history.append(new_agent)
agents_history[iteration] = np.clip(iteration_history, bounds[0],
bounds[1])
fitness_history[iteration] = [
evaluate(function, agent) for agent in agents
]
iteration += 1
if (max_iterations != None) and (max_iterations == iteration):
end = True
if (target_fitness != None) and (target_fitness == best_fitness):
end = True
# calculate total fitness for all agents
fitnesses = np.array([[evaluate(function, agent) for agent in agents]
for agents in agents_history])
best_agent_id = int(np.argmin(fitnesses) / max_iterations) % population
best_agent = agents_history[:, best_agent_id, :]
best_fitness = fitnesses[:, best_agent_id]
return best_agent, best_fitness, agents_history #best_fitness_history
i, :].argsort()[-args.re_ranking:][::-1].tolist()
_, activations_2nd = capsule_net(data[i, :].unsqueeze(0),
candidate_labels)
Y_pred[i, candidate_labels] = activations_2nd.squeeze(
2).data.cpu().numpy()
for i in range(Y_pred.shape[0]):
sorted_idx = np.argpartition(-Y_pred[i, :], top_k)[:top_k]
row_idx_list += [i + start_idx] * top_k
col_idx_list += (sorted_idx).tolist()
val_idx_list += Y_pred[i, sorted_idx].tolist()
done = time.time()
elapsed = done - start
print("\r Reranking: {} Iteration: {}/{} ({:.1f}%) Loss: {:.5f} {:.5f}".
format(args.re_ranking, batch_idx, nr_batches,
batch_idx * 100 / nr_batches, 0, elapsed),
end="")
m = max(row_idx_list) + 1
n = max(k_trn, k_tst)
print(elapsed)
Y_tst_pred = sp.csr_matrix((val_idx_list, (row_idx_list, col_idx_list)),
shape=(m, n))
if k_trn >= k_tst:
Y_tst_pred = Y_tst_pred[:, :k_tst]
evaluate(Y_tst_pred.toarray(), Y_tst)
def rnn_cv( folds, n_hidden=10, n_epochs=50, lr=0.001, lrd = 0.999, reg_coef= 0.01, doSmoothing=False, useEssentia=False):
doSaveModel = False
if doSmoothing:
dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g_smoothed'%(nb_features, n_hidden, n_epochs, lr, reg_coef)
else:
dir_name = 'nfeat%d_nh%d_ne%d_lr%g_reg%g'%(nb_features, n_hidden, n_epochs, lr, reg_coef)
MODELDIR = 'rnn/' + dir_name + '/'
LOGDIR = MODELDIR
if not path.exists(MODELDIR):
makedirs(MODELDIR)
print '... output dir: %s'%(MODELDIR)
# smoothing params
taille = 12
wts = np.ones(taille-1)*1./taille
wts = np.hstack((np.array([1./(2*taille)]), wts, np.array([1./(2*taille)])))
delay = (wts.shape[0]-1) / 2
# # initialize global logger variable
# print '... initializing global logger variable'
# logger = logging.getLogger(__name__)
# withFile = False
# logger = settings.init(MODELDIR + 'train.log', withFile)
# perf_file_name = LOGDIR + 'rnn_nh%d_ne%d_lr%g_reg%g.log'%(n_hidden, n_epochs, lr, reg_coef)
perf_file_name = LOGDIR + 'performance.log'
log_f = open(perf_file_name, 'w')
all_fold_pred = list()
all_fold_y_test = list()
all_fold_id_test = list()
for fold_id in range(10):
# fold_id = 0
fold = folds[fold_id]
t0 = time.time()
# print '... loading FOLD %d'%fold_id
# if useEssentia:
# fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed_essentia.pkl'%(fold_id), "rb" ) )
if useEssentia:
X_train = fold['train']['X']
y_train = fold['train']['y']
id_train = fold['train']['song_id']
X_test = fold['test']['X']
y_test = fold['test']['y']
id_test = fold['test']['song_id']
else:
fold = pickle.load( open( DATADIR + '/pkl/fold%d_normed.pkl'%(fold_id), "rb" ) )
X_train, y_train, id_train = load_X_from_fold_to_3dtensor(fold, 'train', NUM_OUTPUT)
X_test, y_test, id_test = load_X_from_fold_to_3dtensor(fold, 'test', NUM_OUTPUT)
print X_train.shape, y_train.shape, X_test.shape, y_test.shape
if useMelodyFeatures:
# first feature = slope, other two = mean, std
melody_train, melody_test = subset_features(all_song_melody_features, id_train, id_test)
# melody_train = melody_train[:,:,1:]
# melody_test = melody_test[:,:,1:]
# standardize train data
melody_concat_train = np.reshape(melody_train, (melody_train.shape[0]*melody_train.shape[1], melody_train.shape[2]), order='C')
melody_concat_train_normed, scaler = standardize(melody_concat_train)
# print concat_train_normed.shape
melody_train_normed = np.reshape(melody_concat_train_normed, (melody_train.shape[0], melody_train.shape[1], melody_train.shape[2]), order='C')
del melody_concat_train, melody_concat_train_normed
# standardize test data
melody_concat_test = np.reshape(melody_test, (melody_test.shape[0]*melody_test.shape[1], melody_test.shape[2]), order='C')
melody_concat_test_normed, _ = standardize(melody_concat_test, scaler)
# print concat_test_normed.shape
melody_test_normed = np.reshape(melody_concat_test_normed, (melody_test.shape[0], melody_test.shape[1], melody_test.shape[2]), order='C')
del melody_concat_test, melody_concat_test_normed
# concat with the other features
X_train = np.concatenate((X_train, melody_train_normed), axis=2)
X_test = np.concatenate((X_test, melody_test_normed), axis=2)
if useTempoFeatures:
tempo_train, tempo_test = subset_features(all_song_tempo_features, id_train, id_test)
# standardize train data
tempo_concat_train = np.reshape(tempo_train, (tempo_train.shape[0]*tempo_train.shape[1], tempo_train.shape[2]), order='C')
tempo_concat_train_normed, scaler = standardize(tempo_concat_train)
# print concat_train_normed.shape
tempo_train_normed = np.reshape(tempo_concat_train_normed, (tempo_train.shape[0], tempo_train.shape[1], tempo_train.shape[2]), order='C')
del tempo_concat_train, tempo_concat_train_normed
# standardize test data
tempo_concat_test = np.reshape(tempo_test, (tempo_test.shape[0]*tempo_test.shape[1], tempo_test.shape[2]), order='C')
tempo_concat_test_normed, _ = standardize(tempo_concat_test, scaler)
# print concat_test_normed.shape
tempo_test_normed = np.reshape(tempo_concat_test_normed, (tempo_test.shape[0], tempo_test.shape[1], tempo_test.shape[2]), order='C')
del tempo_concat_test, tempo_concat_test_normed
# concat with the other features
X_train = np.concatenate((X_train, tempo_train_normed), axis=2)
X_test = np.concatenate((X_test, tempo_test_normed), axis=2)
# print id_test.shape
# X_train = X_train[0:100,:,:]
# y_train = y_train[0:100,:,:]
# X_train = X_train[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]]
# X_test = X_test[:,[10,12,13,17,19,82,83,84,85,89,90,91,103,140,142,146,148,212,214,218,220]]
# X_train = X_train[:,[13,85,103,142,214]]
# X_test = X_test[:,[13,85,103,142,214]]
# X_test = X_train[119:119+y_test.shape[0],:]
# y_test = y_train[119:119+y_test.shape[0]]
print X_train.shape, y_train.shape, X_test.shape, y_test.shape
nb_seq_train, nb_frames_train, nb_features_train = X_train.shape
nb_seq_test, nb_frames_test, nb_features_test = X_test.shape
assert nb_frames_train == nb_frames_test, 'ERROR: nb of frames differ from TRAIN to TEST'
assert nb_features_train == nb_features_test, 'ERROR: nb of features differ from TRAIN to TEST'
dim_ouput_train = y_train.shape[2]
dim_ouput_test = y_test.shape[2]
assert dim_ouput_test == dim_ouput_train, 'ERROR: nb of targets differ from TRAIN to TEST'
n_in = nb_features_train
n_out = dim_ouput_train
n_steps = nb_frames_train
validation_frequency = nb_seq_train * 2 # for logging during training: every 2 epochs
model = rnn_model.MetaRNN(n_in=n_in, n_hidden=n_hidden, n_out=n_out,
learning_rate=lr, learning_rate_decay=lrd,
L1_reg=reg_coef, L2_reg=reg_coef,
n_epochs=n_epochs, activation='tanh')
model.fit(X_train, y_train, validation_frequency=validation_frequency)
if doSaveModel:
# model_name = MODELDIR + 'rnn_fold%d_nh%d_nepochs%d_lr%g_reg%g.pkl'%(fold_id, n_hidden, n_epochs, lr, reg_coef)
model_name = MODELDIR + 'model_fold%d.pkl'%(fold_id)
model.save(fpath=model_name)
pred = list()
for ind_seq_test in xrange(nb_seq_test):
pred.append(model.predict(X_test[ind_seq_test]))
y_hat = np.array(pred, dtype=float)
print y_hat.shape
if doSmoothing:
# smooooooth
y_hat_smooth = np.zeros_like(y_hat, dtype=float)
for i in xrange(y_hat.shape[0]):
y_hat_smooth[i, :, 0] = np.convolve(y_hat[i, :, 0], wts, mode='same')
y_hat_smooth[i, :delay, 0] = y_hat[i, :delay, 0]
y_hat_smooth[i, -delay:, 0] = y_hat[i, -delay:, 0]
y_hat_smooth[i, :, 1] = np.convolve(y_hat[i, :, 1], wts, mode='same')
y_hat_smooth[i, :delay, 1] = y_hat[i, :delay, 1]
y_hat_smooth[i, -delay:, 1] = y_hat[i, -delay:, 1]
# save predictions on the test subset, before reshaping to 2-d arrays (I need 3d arrays)
if doSmoothing:
# fold_pred = [item for sublist in fold_pred for item in sublist]
# fold_pred = np.array(fold_pred, dtype=float)
pred_file = LOGDIR + 'fold%d_test_predictions.pkl'%(fold_id)
pickle.dump( y_hat_smooth, open( pred_file, "wb" ) )
print ' ... predictions y_hat_smooth saved in: %s'%(pred_file)
else:
# fold_pred = [item for sublist in fold_pred for item in sublist]
# fold_pred = np.array(fold_pred, dtype=float)
pred_file = LOGDIR + 'fold%d_test_predictions.pkl'%(fold_id)
pickle.dump( y_hat, open( pred_file, "wb" ) )
print ' ... predictions y_hat saved in: %s'%(pred_file)
if doSmoothing:
y_hat_smooth = np.reshape(y_hat_smooth, (y_hat_smooth.shape[0]*y_hat_smooth.shape[1], y_hat_smooth.shape[2]))
y_hat = np.reshape(y_hat, (y_hat.shape[0]*y_hat.shape[1], y_hat.shape[2]))
y_test_concat = np.reshape(y_test, (y_test.shape[0]*y_test.shape[1], y_test.shape[2]))
print y_hat.shape, y_test_concat.shape
assert y_hat.shape == y_test_concat.shape, 'ERROR: pred and ref shapes are different!'
# concat hyp labels:
if doSmoothing:
all_fold_pred.append(y_hat_smooth.tolist())
else:
all_fold_pred.append(y_hat.tolist())
# concat ref labels:
all_fold_y_test.append(y_test_concat.tolist())
if doSmoothing:
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat_smooth, id_test.shape[0])
else:
RMSE, pcorr, error_per_song, mean_per_song = evaluate(y_test_concat, y_hat, id_test.shape[0])
s = (
'fold: %d valence: %.4f %.4f arousal: %.4f %.4f\n'
% (fold_id, RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0])
)
print s
log_f.write(s)
# predict on the train set and save predictions (useful to train rnn2)
if doSmoothing:
pred = list()
for ind_seq_train in xrange(nb_seq_train):
pred.append(model.predict(X_train[ind_seq_train]))
train_y_hat = np.array(pred, dtype=float)
print train_y_hat.shape
train_y_hat_smooth = np.zeros_like(train_y_hat, dtype=float)
for i in xrange(train_y_hat.shape[0]):
train_y_hat_smooth[i, :, 0] = np.convolve(train_y_hat[i, :, 0], wts, mode='same')
train_y_hat_smooth[i, :delay, 0] = train_y_hat[i, :delay, 0]
train_y_hat_smooth[i, -delay:, 0] = train_y_hat[i, -delay:, 0]
train_y_hat_smooth[i, :, 1] = np.convolve(train_y_hat[i, :, 1], wts, mode='same')
train_y_hat_smooth[i, :delay, 1] = train_y_hat[i, :delay, 1]
train_y_hat_smooth[i, -delay:, 1] = train_y_hat[i, -delay:, 1]
# no reshape, I need 3d arrays
# train_y_hat_smooth = np.reshape(train_y_hat_smooth, (train_y_hat_smooth.shape[0]*train_y_hat_smooth.shape[1], train_y_hat_smooth.shape[2]))
pred_file = LOGDIR + 'fold%d_train_predictions.pkl'%(fold_id)
pickle.dump( train_y_hat_smooth, open( pred_file, "wb" ) )
print ' ... predictions y_hat_smooth saved in: %s'%(pred_file)
else:
pred = list()
for ind_seq_train in xrange(nb_seq_train):
pred.append(model.predict(X_train[ind_seq_train]))
train_y_hat = np.array(pred, dtype=float)
pred_file = LOGDIR + 'fold%d_train_predictions.pkl'%(fold_id)
pickle.dump( train_y_hat, open( pred_file, "wb" ) )
print ' ... predictions y_hat saved in: %s'%(pred_file)
doPlot = False
if doPlot:
fig, ax = plt.subplots()
x1 = np.linspace(1, y_test_concat.shape[0], y_test_concat.shape[0])
if EMO == 'valence':
ax.plot(x1, y_test_concat[:, 0], 'o', label="Data")
# ax.plot(x1, y_hat[:,0], 'r-', label="OLS prediction")
ax.plot(x1, y_hat[:,0], 'ro', label="OLS prediction")
else:
ax.plot(x1, y_test_concat[:, 1], 'o', label="Data")
ax.plot(x1, y_hat[:,1], 'ro', label="OLS prediction")
plt.title(EMO + ' on Test subset')
ax.legend(loc="best")
plt.show()
# plt.savefig('figures/rnn_%s_fold%d.png'%(EMO, fold_id), format='png')
doPlotTrain = False
if doPlotTrain:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_train[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_train[0])
guess = model.predict(X_train[0])
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
doPlotTest = False
if doPlotTest:
# plt.close('all')
fig = plt.figure()
ax1 = plt.subplot(211)
plt.plot(X_test[0])
ax1.set_title('input')
ax2 = plt.subplot(212)
true_targets = plt.plot(y_test[0])
# guess = model.predict(X_test[0])
guess = y_hat[0]
guessed_targets = plt.plot(guess, linestyle='--')
for i, x in enumerate(guessed_targets):
x.set_color(true_targets[i].get_color())
ax2.set_title('solid: true output, dashed: model output')
plt.show()
print "... Elapsed time: %f" % (time.time() - t0)
all_fold_pred = [item for sublist in all_fold_pred for item in sublist]
all_fold_y_test = [item for sublist in all_fold_y_test for item in sublist]
all_fold_pred = np.array(all_fold_pred, dtype=float)
all_fold_y_test = np.array(all_fold_y_test, dtype=float)
print all_fold_pred.shape, all_fold_y_test.shape
# save predictions
pred_file = LOGDIR + 'all_predictions.pkl'
pickle.dump( all_fold_pred, open( pred_file, "wb" ) )
print ' ... all predictions saved in: %s'%(pred_file)
# ref_file = 'rnn/all_groundtruth.pkl'
# pickle.dump( all_fold_y_test, open( ref_file, "wb" ) )
# compute t-test p-values with baseline predictions
baseline_prediction_file = 'rnn/all_baseline_predictions_260feat.pkl'
baseline_preds = pickle.load(open( baseline_prediction_file, 'r' ))
pvalue_val = stats.ttest_ind(baseline_preds[:,0], all_fold_pred[:,0])[1]
pvalue_ar = stats.ttest_ind(baseline_preds[:,1], all_fold_pred[:,1])[1]
pvalues = (pvalue_val, pvalue_ar)
RMSE, pcorr, error_per_song, mean_per_song = evaluate(all_fold_y_test, all_fold_pred, 0)
# print(
# 'sklearn --> valence: %.4f, arousal: %.4f\n'
# 'Pearson Corr --> valence: %.4f, arousal: %.4f \n'
# # % (RMSE[0], -1. , pcorr[0][0], -1)
# % (RMSE[0],RMSE[1],pcorr[0][0], pcorr[1][0])
# )
s = (
'allfolds valence: %.4f %.4f arousal: %.4f %.4f p-values: %.4f, %.4f\n'
% (RMSE[0], pcorr[0][0], RMSE[1], pcorr[1][0], pvalue_val, pvalue_ar)
)
print s
log_f.write(s)
log_f.close()
return RMSE, pcorr, pvalues
def validate(val_loader, net, criterion, optimizer, epoch, train_args, restore,
visualize):
net.eval()
val_loss = AverageMeter()
inputs_all, gts_all, predictions_all = [], [], []
for vi, data in enumerate(val_loader):
inputs, gts = data
N = inputs.size(0)
inputs = Variable(inputs, volatile=True).cuda()
gts = Variable(gts, volatile=True).cuda()
outputs = net(inputs)
predictions = outputs.data.max(1)[1].squeeze_(1).squeeze_(
0).cpu().numpy()
val_loss.update(criterion(outputs, gts).data[0] / N, N)
if random.random() > train_args['val_img_sample_rate']:
inputs_all.append(None)
else:
inputs_all.append(inputs.data.squeeze_(0).cpu())
gts_all.append(gts.data.squeeze_(0).cpu().numpy())
predictions_all.append(predictions)
acc, acc_cls, mean_iu, fwavacc = evaluate(predictions_all, gts_all,
voc.num_classes)
if mean_iu > train_args['best_record']['mean_iu']:
train_args['best_record']['val_loss'] = val_loss.avg
train_args['best_record']['epoch'] = epoch
train_args['best_record']['acc'] = acc
train_args['best_record']['acc_cls'] = acc_cls
train_args['best_record']['mean_iu'] = mean_iu
train_args['best_record']['fwavacc'] = fwavacc
snapshot_name = 'epoch_%d_loss_%.5f_acc_%.5f_acc-cls_%.5f_mean-iu_%.5f_fwavacc_%.5f_lr_%.10f' % (
epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc,
optimizer.param_groups[1]['lr'])
# torch.save(net.state_dict(), os.path.join(ckpt_path, exp_name, snapshot_name + '.pth'))
# torch.save(optimizer.state_dict(), os.path.join(ckpt_path, exp_name, 'opt_' + snapshot_name + '.pth'))
if train_args['val_save_to_img_file']:
to_save_dir = os.path.join(ckpt_path, exp_name, str(epoch))
check_mkdir(to_save_dir)
val_visual = []
for idx, data in enumerate(zip(inputs_all, gts_all, predictions_all)):
if data[0] is None:
continue
input_pil = restore(data[0])
gt_pil = voc.colorize_mask(data[1])
predictions_pil = voc.colorize_mask(data[2])
if train_args['val_save_to_img_file']:
input_pil.save(os.path.join(to_save_dir, '%d_input.png' % idx))
predictions_pil.save(
os.path.join(to_save_dir, '%d_prediction.png' % idx))
gt_pil.save(os.path.join(to_save_dir, '%d_gt.png' % idx))
val_visual.extend([
visualize(input_pil.convert('RGB')),
visualize(gt_pil.convert('RGB')),
visualize(predictions_pil.convert('RGB'))
])
val_visual = torch.stack(val_visual, 0)
val_visual = vutils.make_grid(val_visual, nrow=3, padding=5)
writer.add_image(snapshot_name, val_visual)
print(
'--------------------------------------------------------------------')
print(
'[epoch %d], [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f]'
% (epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc))
print(
'best record: [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f], [epoch %d]'
% (train_args['best_record']['val_loss'],
train_args['best_record']['acc'],
train_args['best_record']['acc_cls'],
train_args['best_record']['mean_iu'],
train_args['best_record']['fwavacc'],
train_args['best_record']['epoch']))
print(
'--------------------------------------------------------------------')
writer.add_scalar('val_loss', val_loss.avg, epoch)
writer.add_scalar('acc', acc, epoch)
writer.add_scalar('acc_cls', acc_cls, epoch)
writer.add_scalar('mean_iu', mean_iu, epoch)
writer.add_scalar('fwavacc', fwavacc, epoch)
writer.add_scalar('lr', optimizer.param_groups[1]['lr'], epoch)
net.train()
return val_loss.avg
# 1. train model with training data, and recommend samples for labeling
out = AL.process(X_train, y_train, X_test, Nrecs, 'SVM')
IDtoLabel = out['IDtoLabel']
relevanceScore = out['relevanceScore']
# 2. assume get the expert feedback
y_newlabel = y_test[IDtoLabel]
# For Simulation purpose
# performance evaluation: calculate prediction accuracy
classifier = out['classifier']
y_test_predicted = classifier.predict(X_test)
out = evaluate(y_test, y_test_predicted)
accuracy.append(out['accuracy'])
# updated y_test
mask = np.ones(len(y_test), dtype=bool) # all elements included/True.
mask[IDtoLabel] = False
y_test = y_test[mask]
# 3. update training data set for re-train model
out = AL.add_answers(X_train, X_test, y_train, y_newlabel, IDtoLabel)
X_train = out['X_train']
y_train = out['y_train']
X_test = out['X_test']
def main(args):
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
print(args)
print(
'Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/'
)
tb_writer = SummaryWriter()
if os.path.exists("./weights") is False:
os.makedirs("./weights")
train_images_path, train_images_label, val_images_path, val_images_label = read_split_data(
args.data_path)
img_size = {
"s": [300, 384], # train_size, val_size
"m": [384, 480],
"l": [384, 480]
}
num_model = "s"
data_transform = {
"train":
transforms.Compose([
transforms.RandomResizedCrop(img_size[num_model][0]),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
]),
"val":
transforms.Compose([
transforms.Resize(img_size[num_model][1]),
transforms.CenterCrop(img_size[num_model][1]),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
}
# 实例化训练数据集
train_dataset = MyDataSet(images_path=train_images_path,
images_class=train_images_label,
transform=data_transform["train"])
# 实例化验证数据集
val_dataset = MyDataSet(images_path=val_images_path,
images_class=val_images_label,
transform=data_transform["val"])
batch_size = args.batch_size
nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0,
8]) # number of workers
print('Using {} dataloader workers every process'.format(nw))
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=batch_size,
shuffle=True,
pin_memory=True,
num_workers=nw,
collate_fn=train_dataset.collate_fn)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=batch_size,
shuffle=False,
pin_memory=True,
num_workers=nw,
collate_fn=val_dataset.collate_fn)
# 如果存在预训练权重则载入
model = create_model(num_classes=args.num_classes).to(device)
if args.weights != "":
if os.path.exists(args.weights):
weights_dict = torch.load(args.weights, map_location=device)
load_weights_dict = {
k: v
for k, v in weights_dict.items()
if model.state_dict()[k].numel() == v.numel()
}
print(model.load_state_dict(load_weights_dict, strict=False))
else:
raise FileNotFoundError("not found weights file: {}".format(
args.weights))
# 是否冻结权重
if args.freeze_layers:
for name, para in model.named_parameters():
# 除head外，其他权重全部冻结
if "head" not in name:
para.requires_grad_(False)
else:
print("training {}".format(name))
pg = [p for p in model.parameters() if p.requires_grad]
optimizer = optim.SGD(pg, lr=args.lr, momentum=0.9, weight_decay=1E-4)
# Scheduler https://arxiv.org/pdf/1812.01187.pdf
lf = lambda x: ((1 + math.cos(x * math.pi / args.epochs)) / 2) * (
1 - args.lrf) + args.lrf # cosine
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
for epoch in range(args.epochs):
# train
train_loss, train_acc = train_one_epoch(model=model,
optimizer=optimizer,
data_loader=train_loader,
device=device,
epoch=epoch)
scheduler.step()
# validate
val_loss, val_acc = evaluate(model=model,
data_loader=val_loader,
device=device,
epoch=epoch)
tags = [
"train_loss", "train_acc", "val_loss", "val_acc", "learning_rate"
]
tb_writer.add_scalar(tags[0], train_loss, epoch)
tb_writer.add_scalar(tags[1], train_acc, epoch)
tb_writer.add_scalar(tags[2], val_loss, epoch)
tb_writer.add_scalar(tags[3], val_acc, epoch)
tb_writer.add_scalar(tags[4], optimizer.param_groups[0]["lr"], epoch)
torch.save(model.state_dict(), "./weights/model-{}.pth".format(epoch))
def validate(val_loader, net, criterion, optimizer, epoch, train_args, visualize):
# the following code is written assuming that batch size is 1
net.eval()
val_loss = AverageMeter()
gts_all = np.zeros((len(val_loader), args['shorter_size'], 2 * args['shorter_size']), dtype=int)
predictions_all = np.zeros((len(val_loader), args['shorter_size'], 2 * args['shorter_size']), dtype=int)
for vi, data in enumerate(val_loader):
input, gt, slices_info = data
assert len(input.size()) == 5 and len(gt.size()) == 4 and len(slices_info.size()) == 3
input.transpose_(0, 1)
gt.transpose_(0, 1)
slices_info.squeeze_(0)
assert input.size()[3:] == gt.size()[2:]
count = torch.zeros(args['shorter_size'], 2 * args['shorter_size']).cuda()
output = torch.zeros(voc.num_classes, args['shorter_size'], 2 * args['shorter_size']).cuda()
slice_batch_pixel_size = input.size(1) * input.size(3) * input.size(4)
for input_slice, gt_slice, info in zip(input, gt, slices_info):
input_slice = Variable(input_slice).cuda()
gt_slice = Variable(gt_slice).cuda()
output_slice = net(input_slice)
assert output_slice.size()[2:] == gt_slice.size()[1:]
assert output_slice.size()[1] == voc.num_classes
output[:, info[0]: info[1], info[2]: info[3]] += output_slice[0, :, :info[4], :info[5]].data
gts_all[vi, info[0]: info[1], info[2]: info[3]] += gt_slice[0, :info[4], :info[5]].data.cpu().numpy()
count[info[0]: info[1], info[2]: info[3]] += 1
val_loss.update(criterion(output_slice, gt_slice).data[0], slice_batch_pixel_size)
output /= count
gts_all[vi, :, :] /= count.cpu().numpy().astype(int)
predictions_all[vi, :, :] = output.max(0)[1].squeeze_(0).cpu().numpy()
print('validating: %d / %d' % (vi + 1, len(val_loader)))
acc, acc_cls, mean_iu, fwavacc = evaluate(predictions_all, gts_all, voc.num_classes)
train_args['best_record']['val_loss'] = val_loss.avg
train_args['best_record']['epoch'] = epoch
train_args['best_record']['acc'] = acc
train_args['best_record']['acc_cls'] = acc_cls
train_args['best_record']['mean_iu'] = mean_iu
train_args['best_record']['fwavacc'] = fwavacc
snapshot_name = 'epoch_%d_loss_%.5f_acc_%.5f_acc-cls_%.5f_mean-iu_%.5f_fwavacc_%.5f_lr_%.10f' % (
epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc, optimizer.param_groups[1]['lr'])
torch.save(net.state_dict(), os.path.join(ckpt_path, exp_name, snapshot_name + '.pth'))
torch.save(optimizer.state_dict(), os.path.join(ckpt_path, exp_name, 'opt_' + snapshot_name + '.pth'))
if train_args['val_save_to_img_file']:
to_save_dir = os.path.join(ckpt_path, exp_name, str(epoch))
check_mkdir(to_save_dir)
val_visual = []
for idx, data in enumerate(zip(gts_all, predictions_all)):
gt_pil = voc.colorize_mask(data[0])
predictions_pil = voc.colorize_mask(data[1])
if train_args['val_save_to_img_file']:
predictions_pil.save(os.path.join(to_save_dir, '%d_prediction.png' % idx))
gt_pil.save(os.path.join(to_save_dir, '%d_gt.png' % idx))
val_visual.extend([visualize(gt_pil.convert('RGB')),
visualize(predictions_pil.convert('RGB'))])
val_visual = torch.stack(val_visual, 0)
val_visual = vutils.make_grid(val_visual, nrow=2, padding=5)
writer.add_image(snapshot_name, val_visual)
print('-----------------------------------------------------------------------------------------------------------')
print('[epoch %d], [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f]' % (
epoch, val_loss.avg, acc, acc_cls, mean_iu, fwavacc))
print('best record: [val loss %.5f], [acc %.5f], [acc_cls %.5f], [mean_iu %.5f], [fwavacc %.5f], [epoch %d]' % (
train_args['best_record']['val_loss'], train_args['best_record']['acc'], train_args['best_record']['acc_cls'],
train_args['best_record']['mean_iu'], train_args['best_record']['fwavacc'], train_args['best_record']['epoch']))
print('-----------------------------------------------------------------------------------------------------------')
writer.add_scalar('val_loss', val_loss.avg, epoch)
writer.add_scalar('acc', acc, epoch)
writer.add_scalar('acc_cls', acc_cls, epoch)
writer.add_scalar('mean_iu', mean_iu, epoch)
writer.add_scalar('fwavacc', fwavacc, epoch)
net.train()
return val_loss.avg
def run_latency_per_round(train_loader,
test_loader,
comm_matrix,
num_rounds,
epochs,
num_clients,
latency_nodes,
latency_rounds,
net='net',
optimizer='sgd',
lr=0.1):
"""
Runs a decentralized optimization algorithm for the given learning rate for a
number of rounds, over some network. Some nodes send their weights with a one-rounds
latency, only during specific rounds. Outputs the accuracies and returns them.
Params:
train_loader (array): the list of all train datasets, one per client
test_loader (array): the list of test datasets, one per client
comm_matrix (numpy.array): the communication matric modeling the network
num_rounds (int): the number of data exchanges between nodes
epochs (int): the number of optimization steps between each communication (minimum 1)
num_clients (int): the number of clients in the network
latency_nodes (array): the list of delayed nodes
latency_rounds (array): the rounds at which latency will occur across the network
net (string): the neural network framework we use
optimizer (string): the chosen optimizer, SGD by default
lr (double): the learning rate for the optimizaion algorithm
Returns:
global_model (nn.Module): the final global neural network averaging all the clients
client_models (array of Net): the list of all the final client neural networks
accs (array): the corresponding accuracies, with the same shape as lrs
"""
accs = []
global_model, client_models = model_init(num_clients, net)
opt = optimizer_init(client_models, lr, optimizer)
loss, test_loss, acc = 0.0, 0.0, 0.0
for r in range(num_rounds):
old_client_models = client_models
# client update
for i in range(num_clients):
loss += client_update(client_models[i],
opt[i],
train_loader[i],
epoch=epochs)
# diffuse params
if (r in latency_rounds):
diffuse_params_latency(client_models, comm_matrix, latency_nodes)
print("round {}, delay".format(r))
elif (r in latency_rounds + 1):
diffuse_params(client_models, comm_matrix)
diffuse_params_latency(
old_client_models, comm_matrix,
np.setdiff1d(np.array(range(num_clients)), latency_nodes))
print("round {}, delay recovery".format(r))
else:
diffuse_params(client_models, comm_matrix)
print("round {}, normal".format(r))
average_models(global_model, client_models)
test_loss, acc = evaluate(global_model, test_loader)
print('%d-th round' % r)
print('average train loss %0.3g | test loss %0.3g | test acc: %0.3f' %
(loss / num_clients, test_loss, acc))
accs.append(acc)
return global_model, client_models, accs
