def train(train_data,
val_data,
user_list_train_filtered,
user_list_val_filtered,
user_beta_train,
user_beta_val,
k,
dataset,
eta=0.1,
lamb=0.1,
tolerance=1e-4,
num_iter_val=5,
num_total_iter_training=6,
random_seed=786,
kU=None,
cv_flag=True,
verbose=False):
np.random.seed(random_seed)
user_feat = val_data.drop(['user', 'label'], axis=1).values
user_feat_train = train_data.drop(['user', 'label'], axis=1).values
w = np.random.normal(0, 1, user_feat.shape[1])
metrics = Metrics()
metrics.eta_lr = eta
metrics.lamb_reg = lamb
print("running for eta", eta, "and lambda", lamb)
for i in range(num_total_iter_training):
grad, loss = subgradient(w, train_data, user_list_train_filtered,
user_beta_train, k)
grad += lamb * w
w = w - (eta / np.sqrt(i + 1)) * grad
metrics.w_list.append(w)
metrics.loss_opt_list_train.append(loss)
y_scores = user_feat_train.dot(w)
data_true = deepcopy(train_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.micro_auc_rel_k_list_train.append(
compute_micro(data_true, user_list_train_filtered, user_beta_train,
w, k))
if verbose:
print('Epoch', i + 1, 'completed out of', num_total_iter_training,
'for prec@k loss train:', metrics.loss_opt_list_train[-1])
print('Epoch', i + 1, 'completed out of', num_total_iter_training,
'for prec@k grad train:', np.linalg.norm(grad))
# evaluate combined weights
if (cv_flag):
if i % num_iter_val == 0:
y_scores = user_feat.dot(w)
data_true = deepcopy(val_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.micro_auc_rel_k_list_val.append(
compute_micro(data_true, user_list_val_filtered,
user_beta_val, w, k))
if verbose:
print("\n")
print('Epoch', i + 1, 'completed out of',
num_total_iter_training, 'for prec@k loss val:',
metrics.micro_auc_rel_k_list_val[-1])
print("\n")
return metrics, None
def train(train_data,
val_data,
user_list_train_filtered,
user_list_val_filtered,
user_beta_train,
user_beta_val,
k,
eta=0.1,
lamb=0.1,
num_iter_val=5,
num_total_iter_training=6,
n_classifiers=5,
random_seed=786,
verbose=False):
np.random.seed(random_seed)
user_list_val_filtered = user_list_train_filtered[
0:int(0.2 * len(user_list_train_filtered))]
user_list_train_filtered = list(
set(user_list_train_filtered) - set(user_list_val_filtered))
val_data = train_data[train_data['user'].isin(user_list_val_filtered)]
train_data = train_data[train_data['user'].isin(user_list_train_filtered)]
metrics = Metrics()
metrics.eta_lr = eta
metrics.lamb_reg = lamb
classifier_list = []
kf = KFold(n_splits=n_classifiers, shuffle=True)
features = train_data.drop(['user', 'label'], axis=1)
labels = train_data['label']
for _, split_indices in kf.split(features):
split_features = features.iloc[split_indices].values
split_labels = labels.iloc[split_indices].values
num_examples = split_features.shape[0]
w = np.random.normal(0, 1, (split_features.shape[1], ))
w = w / np.linalg.norm(w)
for num_iter in np.arange(num_total_iter_training):
scores = sigmoid(np.dot(split_features, w))
loss = -1 / num_examples * np.sum(split_labels * np.log(scores) +
(1 - split_labels) *
np.log(1 - scores))
print("loss is ", loss)
dLdwx = (scores - split_labels) * scores * (1 - scores)
grad = 1 / num_examples * np.sum(
dLdwx.reshape(-1, 1) * split_features)
grad += lamb * w
print("grad is ", np.linalg.norm(grad))
print("\n")
w = w - (eta / np.sqrt(num_iter + 1)) * grad
accuracy = np.sum(split_labels * (scores > 0.5) + (1 - split_labels) *
(scores < 0.5))
print('accuracy: {}'.format(accuracy / num_examples))
classifier_list.append(w)
print('eta is ', eta, 'and lambda is ', lamb)
print('\n')
classifiers_with_metrics = []
for w in classifier_list:
user_feat = val_data.drop(['user', 'label'], axis=1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(val_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metric = compute_micro(data_true, user_list_val_filtered,
user_beta_train, w, k)
classifiers_with_metrics.append((metric, w))
classifiers_with_metrics.sort(reverse=True, key=lambda x: x[0])
combined_w = classifiers_with_metrics[0][1]
for _, w in classifiers_with_metrics[1:]:
combined_w = merge_micro(val_data, combined_w, w,
user_list_val_filtered, user_beta_train, k)
# create dummy metrics
# need weights and one validation loss for the "best iter" logic
metrics = Metrics()
metrics.w_list.append(combined_w)
metrics.micro_auc_rel_k_list_val.append(0)
metrics.micro_auc_rel_k_list_train.append(0)
metrics.loss_opt_list_train.append(0)
return metrics, None
def train(
train_data,
val_data,
user_list_train_filtered,
user_list_val_filtered,
surr,
eta=0.1,
momentum=0.0,
lamb=0.1,
num_iter_val=5,
tolerance=1e-6,
num_total_iter_training=151,
draw=False,
verbose=True,
random_seed = 786,
w=None):
np.random.seed(random_seed)
metrics = utils.Metrics()
metrics.eta_lr = eta
metrics.lamb_reg = lamb
if w is None:
w = np.random.normal(0, 1, (train_data.shape[1] - 2, ))
prev_grad_w = np.zeros((train_data.shape[1] - 2, ))
for num_iter in np.arange(num_total_iter_training):
tic = time.time()
metrics.w_list.append(w)
loss_opt = 0
k_minus_w_opt = 0
grad_w = np.zeros((train_data.shape[1] - 2, ))
sorting_time = 0
surrogate_time = 0
for user_id in user_list_train_filtered:
user_df_train = train_data[train_data['user'] == user_id]
if(len(user_df_train.label.unique()) == 1):
if(user_df_train.label.iloc[0] == 1.0):
loss_opt += 0.0
else:
loss_opt += 1.0
else:
sorting_start_time = time.time()
beta = int(user_beta_train[user_id])
user_df_pos = user_df_train[user_df_train['label'] == 1]
user_df_neg = user_df_train[user_df_train['label'] == 0]
user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values
user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values
indices_pos, scores_pos = sort_order(user_feat_pos, w)
indices_neg, scores_neg = sort_order(user_feat_neg, w)
sorted_user_feat_pos = user_feat_pos[indices_pos, :]
sorted_user_feat_neg = user_feat_neg[indices_neg, :]
sorted_scores_pos = scores_pos[indices_pos]
sorted_scores_neg = scores_neg[indices_neg]
sorting_time += time.time() - sorting_start_time
surrogate_start_time = time.time()
pi_opt, score_mat = surr.compute_pi(
sorted_scores_pos, sorted_scores_neg,
w, k, beta)
loss_opt_user, _, _, _ = surr.loss(
pi_opt, sorted_scores_pos, sorted_scores_neg, k, beta)
if draw and user_id == 0:
plt.subplot(1,2,1)
plt.imshow(score_mat)
plt.subplot(1,2,2)
plt.imshow(pi_opt)
plt.show()
grad_w_user = surr.gradient(
sorted_user_feat_pos, sorted_user_feat_neg, pi_opt, k, beta)
surrogate_time += time.time() - surrogate_start_time
grad_w_user += lamb*w
loss_opt += loss_opt_user
grad_w += grad_w_user
grad_w = grad_w/len(user_list_train_filtered)
metrics_start_time = time.time()
# sort data once for both micro
user_feat = train_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(train_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.grad_w_list.append(np.linalg.norm(grad_w))
metrics.loss_opt_list_train.append(loss_opt/len(user_list_train_filtered))
metrics.micro_auc_rel_k_list_train.append(utils.compute_micro(data_true, user_list_train_filtered, user_beta_train, w, k))
if verbose:
print(' sorting elapsed time: ', sorting_time)
print(' surrogate elapsed time: ', surrogate_time)
print(' metrics elapsed time: ', time.time() - metrics_start_time)
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'loss train:',metrics.loss_opt_list_train[-1])
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'grad_w:',metrics.grad_w_list[-1])
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'microaucrelk train:',metrics.micro_auc_rel_k_list_train[-1])
else:
print('epoch', num_iter+1, 'completed. micro:{}'.format(metrics.micro_auc_rel_k_list_train[-1]))
if(num_iter%num_iter_val == 0):
loss_opt_val = 0
k_minus_w_opt_val = 0
for user_id in user_list_val_filtered:
user_df_val = val_data[val_data['user'] == user_id]
if(len(user_df_val.label.unique()) == 1):
if(user_df_val.label.iloc[0] == 1.0):
loss_opt_val += 0.0
else:
loss_opt_val += 1.0
else:
beta = int(user_beta_val[user_id])
user_df_pos = user_df_val[user_df_val['label'] == 1]
user_df_neg = user_df_val[user_df_val['label'] == 0]
user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values
user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values
indices_pos, scores_pos = sort_order(user_feat_pos, w)
indices_neg, scores_neg = sort_order(user_feat_neg, w)
sorted_user_feat_pos = user_feat_pos[indices_pos, :]
sorted_user_feat_neg = user_feat_neg[indices_neg, :]
sorted_scores_pos = scores_pos[indices_pos]
sorted_scores_neg = scores_neg[indices_neg]
pi_opt_val, score_mat_val = surr.compute_pi(sorted_scores_pos, sorted_scores_neg, w, k, beta)
loss_opt_user_val, _, _, _ = surr.loss(
pi_opt_val, sorted_scores_pos, sorted_scores_neg, k, beta)
if draw and user_id == 0:
plt.subplot(1,2,1)
plt.imshow(score_mat_val)
plt.subplot(1,2,2)
plt.imshow(pi_opt_val)
plt.show()
loss_opt_val += loss_opt_user_val
# sort data once for both micro
user_feat = val_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(val_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.loss_opt_list_val.append(loss_opt_val/len(user_list_val_filtered))
metrics.micro_auc_rel_k_list_val.append(utils.compute_micro(data_true, user_list_val_filtered, user_beta_val, w, k))
if verbose:
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'loss val:',metrics.loss_opt_list_val[-1])
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'microaucrelk val:',metrics.micro_auc_rel_k_list_val[-1])
else:
print(' val micro:{}'.format(metrics.micro_auc_rel_k_list_val[-1]))
prev_grad_w = momentum * prev_grad_w + (1-momentum) * grad_w
w = w - (eta/np.sqrt(num_iter+1))*(prev_grad_w)
if verbose:
print('Epoch', num_iter+1, ' time taken is: ', time.time() - tic)
print("\n")
# also break if reached tolerance condition
if num_iter >= 10 and max(metrics.loss_opt_list_train[-10:])-min(metrics.loss_opt_list_train[-10:]) <= tolerance:
break
best_iter = (np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]//num_iter_val)*num_iter_val
best_iter = (np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]//num_iter_val)*num_iter_val
best_microaucrelk = metrics.micro_auc_rel_k_list_val[best_iter//num_iter_val]
print('Best micro aucrelk at iter: %d (metric: %f)' % (best_iter, best_microaucrelk))
return metrics, w
def train(
train_data,
val_data,
user_list_train_filtered,
user_list_val_filtered,
surr,
eta=0.1,
momentum=0.0,
lamb=0.1,
num_iter_val=5,
tolerance=1e-4,
num_total_iter_training=21,
draw=False,
verbose=True,
random_seed = 786,
w=None):
np.random.seed(random_seed)
metrics = utils.Metrics()
metrics.eta_lr = eta
metrics.lamb_reg = lamb
if w is None:
w = np.random.normal(0, 1, (train_data.shape[1] - 2, ))
prev_grad_w = np.zeros((train_data.shape[1] - 2, ))
for num_iter in np.arange(num_total_iter_training):
tic = time.time()
metrics.w_list.append(w)
loss_opt = 0
k_minus_w_opt = 0
grad_w = np.zeros((train_data.shape[1] - 2, ))
sorting_time = 0
surrogate_time = 0
pi_opt_avg_user_b = 0
pi_opt_avg_user_n = 0
for user_id in user_list_train_filtered:
user_df_train = train_data[train_data['user'] == user_id]
if(len(user_df_train.label.unique()) == 1):
if(user_df_train.label.iloc[0] == 1.0):
loss_opt += 0.0
else:
loss_opt += 1.0
else:
sorting_start_time = time.time()
beta = int(user_beta_train[user_id])
user_df_pos = user_df_train[user_df_train['label'] == 1]
user_df_neg = user_df_train[user_df_train['label'] == 0]
user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values
user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values
indices_pos, scores_pos = sort_order(user_feat_pos, w)
indices_neg, scores_neg = sort_order(user_feat_neg, w)
sorted_user_feat_pos = user_feat_pos[indices_pos, :]
sorted_user_feat_neg = user_feat_neg[indices_neg, :]
sorted_scores_pos = scores_pos[indices_pos]
sorted_scores_neg = scores_neg[indices_neg]
sorting_time += time.time() - sorting_start_time
surrogate_start_time = time.time()
pi_opt, score_mat = surr.compute_pi(
sorted_scores_pos, sorted_scores_neg,
w, k, beta)
pi_opt_avg_user_b += np.sum(pi_opt)/(beta*k)
pi_opt_avg_user_n += np.sum(pi_opt)/(len(indices_pos)*k)
loss_opt_user, _, _, _ = surr.loss(
pi_opt, sorted_scores_pos, sorted_scores_neg, k, beta)
if draw and user_id == 0:
plt.subplot(1,2,1)
plt.imshow(score_mat)
plt.subplot(1,2,2)
plt.imshow(pi_opt)
plt.show()
grad_w_user = surr.gradient(
sorted_user_feat_pos, sorted_user_feat_neg, pi_opt, k, beta)
surrogate_time += time.time() - surrogate_start_time
grad_w_user += lamb*w
loss_opt += loss_opt_user
grad_w += grad_w_user
grad_w = grad_w/len(user_list_train_filtered)
pi_opt_avg_user_b = pi_opt_avg_user_b/len(user_list_train_filtered)
pi_opt_avg_user_n = pi_opt_avg_user_n/len(user_list_train_filtered)
metrics_start_time = time.time()
# sort data once for both micro
user_feat = train_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(train_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.grad_w_list.append(np.linalg.norm(grad_w))
metrics.loss_opt_list_train.append(loss_opt/len(user_list_train_filtered))
metrics.micro_auc_rel_k_list_train.append(utils.compute_micro(data_true, user_list_train_filtered, user_beta_train, w, k))
if verbose:
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'loss train:',metrics.loss_opt_list_train[-1])
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'grad_w:',metrics.grad_w_list[-1])
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'microaucrelk train:',metrics.micro_auc_rel_k_list_train[-1])
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'pi_opt_avg_user_b:',pi_opt_avg_user_b)
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'pi_opt_avg_user_n:',pi_opt_avg_user_n)
else:
print('k=', k,'Epoch', num_iter+1, 'done. micro:{}'.format(metrics.micro_auc_rel_k_list_train[-1]))
prev_grad_w = momentum * prev_grad_w + (1-momentum) * grad_w
w = w - (eta/np.sqrt(num_iter+1))*(prev_grad_w)
if verbose:
print('Epoch', num_iter+1, ' time taken is: ', time.time() - tic)
print("\n")
# also break if reached tolerance condition
if num_iter >= 10 and max(metrics.loss_opt_list_train[-10:])-min(metrics.loss_opt_list_train[-10:]) <= tolerance:
break
# save output to file
best_iter = np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]
with open('../results/' + dataset + '/test_results/result-testcomp-{}-{}-{}-{}-{}.json'.format(eta, lamb, k, surr.name, random_seed), 'w') as fp:
json.dump(metrics.to_dict(best_iter), fp)
return metrics, w
print('../results/' + dataset + '/test_results/FinalTestResult-testcomp-{}.txt'.format(k))
orig_stdout = sys.stdout
f = open('../results/' + dataset + '/test_results/FinalTestResult-testcomp-{}.txt'.format(k), 'w')
sys.stdout = f
# compute test auc-rel-k at best epoch (from validation)
for name, metric in all_metrics:
best_iter = np.where(np.asarray(metric.loss_opt_list_train)==np.min(metric.loss_opt_list_train))[0][0]
w = metric.w_list[best_iter]
loss_opt_test = 0
k_minus_w_opt_test = 0
# sort data once for both micro
user_feat = test_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(test_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
print('k:', k)
print('Dataset:', dataset)
print('Name:', name)
print(' ', loss_opt_test/len(user_list_test_filtered))
print(' ', utils.compute_micro(data_true, user_list_test_filtered, user_beta_test, w, k))
sys.stdout = orig_stdout
f.close()
print("Total time taken is ", time.time() - tic_all)
# compute val auc-rel-k at best epoch
for name, metric in all_metrics:
best_iter = np.where(
np.asarray(metric.loss_opt_list_train) == np.min(
metric.loss_opt_list_train))[0][0]
w = metric.w_list[best_iter]
loss_opt_val = 0
k_minus_w_opt_val = 0
# sort data once for both micro
user_feat = val_data.drop(['user', 'label'], axis=1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(val_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
print('k:', k)
print('Dataset:', dataset)
print('Name:', name)
print(' ', loss_opt_val / len(user_list_val_filtered))
print(
' ',
utils.compute_micro(data_true, user_list_val_filtered, user_beta_val,
w, k))
sys.stdout = orig_stdout
f.close()