def evaluation():
args = parse()
class_label_dict, label_class_dict = _read_label_file(args.label)
classes = list()
num_classes = len(class_label_dict)
for i in range(num_classes):
classes.append(label_class_dict[i])
gt_dict = utils.read_json(args.gt, class_label_dict)
pred_dict = utils.read_log(args.log, class_label_dict)
y_true, y_pred = utils.gen_yture_ypred(gt_dict, pred_dict)
metrics = utils.Metrics(y_true, y_pred, classes)
conf_matrix = metrics.confusion_matrix
metrics_list = metrics.metrics_list()
# print result
print("class evaluation")
print("~" * 50)
print("accuracy: %.6f" % (metrics_list[-1]))
print("~" * 50)
for i in range(num_classes):
print("%s_recall: %.6f" %
(label_class_dict[i], metrics_list[i * 2]))
print("%s_precision: %.6f" %
(label_class_dict[i], metrics_list[i * 2 + 1]))
print("~" * 50)
print("Confusion Matrix")
print(conf_matrix)
if args.verbose:
metrics.plot_confusion_matrix()
print("Done.")
def evaluate(opt, dloader, model, epoch=0, vis=None, use_saved_file=False):
# Visualizer
opt.save_visuals = True
if vis is None:
if hasattr(opt, 'save_visuals') and opt.save_visuals:
vis = Visualizer(os.path.join(opt.ckpt_path, 'test_log'))
else:
opt.save_visuals = False
model.setup(is_train=False)
metric = utils.Metrics()
results = {}
for step, data in enumerate(dloader):
input, output, input_unocc, output_unocc = data
dec_output, latent, nelbo = model.test(input, output)
# results with partial occlusion in the TOP:
crop_size_1 = opt.crop_size[1]
output_eval = torch.cat([input_unocc, output], dim=1)[:, :, :,
-crop_size_1:]
rec_pred_eval = dec_output[:, :, :, -crop_size_1:]
metric.update(output_eval, rec_pred_eval)
if (step + 1) % opt.log_every == 0:
print('{}/{}'.format(step + 1, len(dloader)))
if opt.save_visuals:
vis.add_images(model.get_visuals(), step, prefix='test_val')
# BCE, MSE
results.update(metric.get_scores())
return results
def evaluate(opt, dloader, model, use_saved_file=False):
# Visualizer
if hasattr(opt, 'save_visuals') and opt.save_visuals:
vis = Visualizer(os.path.join(opt.ckpt_path, 'tb_test'))
else:
opt.save_visuals = False
model.setup(is_train=False)
metric = utils.Metrics()
results = {}
if hasattr(opt, 'save_all_results') and opt.save_all_results:
save_dir = os.path.join(opt.ckpt_path, 'results')
os.makedirs(save_dir, exist_ok=True)
else:
opt.save_all_results = False
# Hacky
is_bouncing_balls = ('bouncing_balls' in opt.dset_name) and opt.n_components == 4
if is_bouncing_balls:
dloader.dataset.return_positions = True
saved_positions = os.path.join(opt.ckpt_path, 'positions.npy') if use_saved_file else ''
velocity_metric = utils.VelocityMetrics(saved_positions)
count = 0
for step, data in enumerate(dloader):
if not is_bouncing_balls:
input, gt = data
else:
input, gt, positions = data
output, latent = model.test(input, gt)
pred = output[:, opt.n_frames_input:, ...]
metric.update(gt, pred)
if opt.save_all_results:
gt = np.concatenate([input.numpy(), gt.numpy()], axis=1)
prediction = utils.to_numpy(output)
count = save_images(prediction, gt, latent, save_dir, count)
if is_bouncing_balls:
# Calculate position and velocity from pose
pose = latent['pose'].data.cpu()
velocity_metric.update(positions, pose, opt.n_frames_input)
if (step + 1) % opt.log_every == 0:
print('{}/{}'.format(step + 1, len(dloader)))
if opt.save_visuals:
vis.add_images(model.get_visuals(), step, prefix='test')
# BCE, MSE
results.update(metric.get_scores())
if is_bouncing_balls:
# Don't break the original code
dloader.dataset.return_positions = False
results.update(velocity_metric.get_scores())
return results
def evaluate(opt, dloader, model, use_saved_file=False):
# Visualizer
if hasattr(opt, 'save_visuals') and opt.save_visuals:
vis = Visualizer(os.path.join(opt.ckpt_path, 'tb_test'))
else:
opt.save_visuals = False
model.setup(is_train=False)
metric = utils.Metrics()
results = {}
if hasattr(opt, 'save_all_results') and opt.save_all_results:
save_dir = os.path.join(opt.ckpt_path, 'results')
os.makedirs(save_dir, exist_ok=True)
else:
opt.save_all_results = False
count = 0
results = []
for step, data in enumerate(dloader):
input, output, neigh, dist, man = data
res_dict = model.test(*data)
results.append(res_dict)
# if opt.save_all_results:
# gt = np.concatenate([input.numpy(), gt.numpy()], axis=1)
# prediction = utils.to_numpy(dec_output)
# count = save_images(prediction, gt, latent)
if (step + 1) % opt.log_every == 0:
print('{}/{}'.format(step + 1, len(dloader)))
# if opt.save_visuals:
# vis.add_images(model.get_visuals(), step, prefix='test')
final_results = {
'Rank_G': 0,
'Trace_K': 0,
'Local_geom': 0,
'Total_loss': 0
}
N = len(results)
for item in results:
final_results['Rank_G'] += item['Rank_G'] / N
final_results['Trace_K'] += item['Trace_K'] / N
final_results['Local_geom'] += item['Local_geom'] / N
final_results['Total_loss'] += item['Total_loss'] / N
# MSE
# results.update(metric.get_scores())
# TODO: metric.reset?
return final_results
def evaluate(opt,
model,
data_loader,
logger,
error_threshold=0.05,
limit=None,
vis=None):
'''
Loop through the dataset and calculate evaluation metrics.
'''
if model.compare_model is not None:
logger.print('Comparison: {} ({}), {} ({})'.format(\
model.iterator.name(), model.iterator.n_operations,
model.compare_model.name(), model.compare_model.n_operations))
logger.print('Initialization: {}'.format(opt.initialization))
logger.print('Error threshold: {}'.format(error_threshold))
metric = utils.Metrics(scale=1, error_threshold=error_threshold)
images = {'error_curves': [], 'results': []}
for step, data in enumerate(data_loader):
bc, gt, x = data['bc'], data['final'], data['x']
f = None if 'f' not in data else data['f']
if opt.initialization != 'random':
# Test time: do not change data if 'random'
x = utils.initialize(x, bc, opt.initialization)
results, x = model.evaluate(x, gt, bc, f, opt.n_evaluation_steps)
# Update metric
metric.update(results)
if step % opt.log_every == 0:
img = utils.plot_error_curves(results, num=4)
if vis is not None:
vis.add_image({'errors_avg_init': img}, step)
images['error_curves'].append(img)
img = utils.plot_results({'x': x, 'gt': gt})
if vis is not None:
vis.add_image({'results': img}, step)
images['results'].append(img)
if (step + 1) % opt.log_every == 0:
print('Step {}'.format(step + 1))
if limit is not None and (step + 1) == limit:
break
# Get results
results = metric.get_results()
for key in results:
logger.print('{}: {}'.format(key, results[key]))
metric.reset()
return results, images
def main():
args = argparser()
# json 格式的 ground truth
gt_dict = utils.read_json(args.gt)
# gt_dict = utils.read_url_list(args.gt)
infered_dict_list = infer.infer(gt_dict, args.tool, ak=args.ak, sk=args.sk)
if args.log:
utils.logs(infered_dict_list, args.log)
y_true, y_pred = utils.get_true_pred(gt_dict, infered_dict_list)
metric = utils.Metrics(y_true, y_pred)
conf_matrix = metric.confusion_matrix()
acc = metric.accuracy()
pulp_recall = metric.pulp_recall()
pulp_precision = metric.pulp_precision()
sexy_recall = metric.sexy_recall()
sexy_precision = metric.sexy_precision()
normal_recall = metric.normal_recall()
normal_precision = metric.normal_precision()
print('\n')
print('【%s】剑皇测试' % __NAME[args.tool])
print('~' * 50)
print('Ground Truth: ')
print('总样本: %d' % len(gt_dict))
print('有效识别样本: %d' % np.sum(conf_matrix))
print('~' * 50)
print('测试集分布： ')
print('%d 个色情样本' % (np.sum(conf_matrix, axis=1)[0]))
print('%d 个性感样本' % (np.sum(conf_matrix, axis=1)[1]))
print('%d 个正常样本' % (np.sum(conf_matrix, axis=1)[2]))
print('~' * 50)
print('模型指标： ')
print('accuracy: %f ' % acc)
print('pulp_recall: %f ' % pulp_recall)
print('pulp_precision: %f ' % pulp_precision)
print('sexy_recall: %f ' % sexy_recall)
print('sexy_precision: %f ' % sexy_precision)
print('normal_recall: %f ' % normal_recall)
print('normal_precision: %f ' % normal_precision)
print('~' * 50)
print('Confusion Matrix: ')
print(conf_matrix)
print('\n')
if args.vis:
metric.plot_confusion_matrix()
def decision_tree_top3_feats_predict_result():
"""
用所有次的 ['乳酸脱氢酶', '超敏C反应蛋白', '淋巴细胞(%)'] 进行预测
孙川 2020.04.07
"""
# 决定是否使用每个病人最后一天的数据
last_sample = False
data1 = pd.read_parquet('data/time_series_375.parquet')[[
'乳酸脱氢酶', '超敏C反应蛋白', '淋巴细胞(%)', '出院时间', '出院方式'
]]
data2 = pd.read_parquet('data/time_series_test_110.parquet')[[
'乳酸脱氢酶', '超敏C反应蛋白', '淋巴细胞(%)', '出院时间', '出院方式'
]]
# data1是375 data2是110 concat是485
for data in [data1, data2, utils.concat_data(data1, data2)]:
# 滑窗合并数据
data = utils.merge_data_by_sliding_window(data,
n_days=1,
dropna=True,
subset=utils.top3_feats_cols,
time_form='diff')
#是否使用最后一次的数据，因为merge_data_by_sliding_window中last自带升序因此这里取first()
#groupby后，每个病人的索引是二级索引t_diff，是升序
if last_sample:
data = data.groupby('PATIENT_ID').first()
# 论文决策树预测
data['pred'] = data.apply(utils.decision_tree, axis=1)
# 调用自己写的结果统计方式utils.Metrics
metrics = utils.Metrics(acc='overall',
f1='overall',
conf_mat='overall',
report='overall')
metrics.record(data['出院方式'], data['pred'])
metrics.print_metrics()
def main():
## get model/training params
args = parser.parse_args()
if args.debug:
print ('==== DEBUGGING MODE ====')
# get name of script for saving models
script_name = os.path.basename(__file__)
## Initialize metrics ###
TrainingEval = utils.TrainingMetrics(script_name)
working_dir = TrainingEval.working_dir
valid = Prepare_Data(args.data,'valid/valid')
valid_batches = DataLoader(valid, args.batch_size,
drop_last=True, shuffle=True)
Validation = utils.Metrics(valid_batches, working_dir ,'validation')
# cp running script to working dir.
os.system('cp {} {}'.format(script_name, working_dir))
## Initialize model
if torch.cuda.is_available():
model = ConvNet(args.kernel_size, args.stride, args.padding,
args.ks_pool, args.str_pool, args.pad_pool).cuda()
else:
model = ConvNet(args.kernel_size, args.stride, args.padding,
args.ks_pool, args.str_pool, args.pad_pool)
## log model/training params to file
LogFile = utils.LogFile(args, model, working_dir)
## Loss and optimizer
criterion = nn.CrossEntropyLoss() # doees not ignore padding (0) ignore_index=0
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
# Train the model
step = -1 # nr of batches
loss_list = []
acc_list = []
valid_loss_list = []
valid_acc_list = []
for epoch in range(args.num_epochs):
for train_ds in range(0,10):
f = args.data
name = 'train/train_{}'.format(train_ds)
train = Prepare_Data(f,name)
train_batches = DataLoader(train, batch_size=args.batch_size,
drop_last=True, shuffle=True)
for i, batch in enumerate(train_batches):
step += 1
# one hot encode
batch = utils.to_one_hot(batch)
# transpose to input seq as vector
batch = torch.transpose(batch,1,2) #transpose dim 1,2 => channels=aa
## Run the forward pass ##
out = model(batch) # sandsynligheder=> skal være [10,25,502] hvor de 25 er sandsynligheder
# convert back to aa labels from one hot for loss
batch_labels = utils.from_one_hot(batch) # integers for labels med 100% sikkerhed
## loss ##
loss = criterion(out, batch_labels)
loss_list.append(loss.item())
## switch model to training mode, clear gradient accumulators ##
model.train()
optimizer.zero_grad()
## Backprop and perform Adam optimisation ##
loss.backward()
optimizer.step()
## Track the accuracy ##
if i % 50 == 0:
# ##########
acc = TrainingEval.get_acc(out,batch_labels)
acc_list.append(acc)
TrainingEval.save_metrics(acc, loss.item(), step, epoch)
print('Epoch [{}/{}], Step: {}, Loss: {:.4f}, Accuracy: {:.4f}%'
.format(epoch + 1, args.num_epochs, step,
loss.item(), acc*100))
# Validation ##
if i % 1000 == 0:
val_loss, val_acc, conf_matrix = \
Validation.get_performance(model,criterion,
confusion_matrix = True)
Validation.save(val_acc, val_loss, epoch, step)
# add to list for fast plotting
valid_loss_list.append(val_loss)
valid_acc_list.append(val_acc)
print('Validation: Loss: {:.4f}, Accuracy: {:.4f}%\n'
.format(val_loss, val_acc*100))
# plot
TrainingEval.plot_metrics(acc_list, loss_list,
valid_acc_list, valid_loss_list, epoch)
Validation.plot_confusion_matrix(conf_matrix)
Validation.plot_per_class(conf_matrix)
# if i % 2000 == 0:
# # Save the model
# TrainingEval.save_model(model.state_dict(), i)
# LogFile.log_saved_model(step)
# Save the model every two train_-ds
if train_ds % 5 ==0:
utils.save_checkpoint(model, optimizer, epoch, train_ds,loss_list, acc_list, working_dir)
utils.save_final_model(model, working_dir)
LogFile.log_saved_model(step)
LogFile.log_performance(acc, loss.item(), ds_type='Training')
if args.testing:
f = args.data
name = 'test/test_1'
test = Prepare_Data(f,name)
test_batches = DataLoader(test, batch_size=args.batch_size,
drop_last=True, shuffle=True)
Test = utils.Metrics(test_batches, working_dir ,'test')
test_loss, test_acc, conf_matrix = Test.get_performance(
model, criterion, confusion_matrix = True)
Test.save(test_acc, test_loss, epoch=-1, step=-1)
Test.plot_confusion_matrix(conf_matrix)
Test.save_conf_matrix(conf_matrix)
Test.plot_per_class(conf_matrix)
LogFile.log_performance(test_acc, test_loss, ds_type='Test')
def train(training_model,
training_data,
opts,
lr_scheduler,
epochs,
optimizer,
validation_function=None):
old_lr = lr_scheduler.get_last_lr()[0]
iterations_per_epoch = len(training_data)
num_instances = opts.popdist_size if opts.use_popdist else 1
metrics = utils.Metrics(running_mean_length=iterations_per_epoch,
distributed=opts.use_popdist)
# Determine loss scaling change points
num_loss_scaling_steps = int(
math.log2(opts.loss_scaling // opts.initial_loss_scaling)) + 1
loss_scaling_steps = {
i * (opts.epoch // num_loss_scaling_steps) + 1:
opts.initial_loss_scaling * (2**i)
for i in range(num_loss_scaling_steps)
}
new_loss_scaling = old_loss_scaling = opts.initial_loss_scaling
for epoch in epochs:
logging.info(f"Epoch {epoch}/{opts.epoch}")
if opts.disable_metrics:
bar = training_data
else:
bar = tqdm(training_data, total=iterations_per_epoch)
epoch_start_time = time.time()
total_sample = 0
if epoch in loss_scaling_steps.keys():
new_loss_scaling = loss_scaling_steps[epoch]
for batch_idx, (input_data, labels) in enumerate(bar):
preds, losses = training_model(input_data, labels)
epoch_num = epoch - 1 + float(batch_idx + 1) / iterations_per_epoch
if not opts.disable_metrics:
with torch.no_grad():
mean_loss = torch.mean(losses).item()
acc = utils.accuracy(preds, labels)
metrics.save_value("accuracy", acc)
metrics.save_value("loss", mean_loss)
aggregated_loss = metrics.get_running_mean("loss")
aggregated_acc = metrics.get_running_mean("accuracy")
bar.set_description(
f"Loss:{aggregated_loss:0.4f} | Accuracy:{aggregated_acc:0.2f}%"
)
total_sample += input_data.size()[0] * num_instances
if not opts.disable_metrics and (
(batch_idx + 1) %
(iterations_per_epoch // opts.logs_per_epoch) == 0):
elapsed_time = metrics.get_elapsed_time()
num_batches = metrics.get_count()
if validation_function is not None and (
epoch % opts.validation_frequency
== 0) and (not opts.use_popdist
or opts.popdist_rank == 0):
training_model.detachFromDevice()
validation_accuracy = validation_function()
model.train()
training_model.attachToDevice()
logging.info(
f"Validation Accuracy: {validation_accuracy:0.2f}%")
else:
validation_accuracy = 0.0
# save metrics
result_dict = {
"loss_avg":
metrics.get_running_mean("loss"),
"loss_batch":
metrics.get_value("loss"),
"epoch":
epoch_num,
"iteration":
batch_idx + 1 + (epoch - 1) * iterations_per_epoch,
"train_accuracy_avg":
metrics.get_running_mean("accuracy"),
"train_accuracy_batch":
metrics.get_value("accuracy"),
"learning_rate":
old_lr,
"loss_scaling":
old_loss_scaling,
"train_img_per_sec":
(num_batches * input_data.size()[0] / elapsed_time),
"latency_sec":
elapsed_time / (num_batches * num_instances),
"validation_accuracy":
validation_accuracy
}
utils.Logger.log_train_results(result_dict)
# lr schedule
lr_scheduler.step(epoch_num)
new_lr = lr_scheduler.get_last_lr()[0]
if new_lr != old_lr or new_loss_scaling != old_loss_scaling:
if new_loss_scaling != old_loss_scaling:
optimizer.loss_scaling = new_loss_scaling
if opts.optimizer == 'sgd':
optimizer.param_groups[0][
"velocity_scaling"] = new_loss_scaling / opts.loss_velocity_scaling_ratio
optimizer.param_groups[1][
"velocity_scaling"] = new_loss_scaling / opts.loss_velocity_scaling_ratio
training_model.setOptimizer(optimizer)
old_lr = new_lr
old_loss_scaling = new_loss_scaling
if opts.lr_schedule == "step":
logging.info(f"Learning rate is changed to {new_lr}")
epoch_end_time = time.time()
if not opts.disable_metrics:
aggregated_acc = metrics.get_running_mean("accuracy")
logging.info(
f"Epoch {epoch}: Train accuracy is {aggregated_acc:0.2f}%")
elapsed_time = epoch_end_time - epoch_start_time
# sync metrics
if opts.use_popdist:
total_sample = utils.sync_metrics(total_sample)
elapsed_time = utils.sync_metrics(elapsed_time)
epoch_throughput = total_sample / elapsed_time
logging.info(
f"Throughput of the epoch:{epoch_throughput:0.1f} img/sec")
# save, in case of multiple processes save the weights only from the first instance.
if not opts.checkpoint_path == "" and (not opts.use_popdist
or opts.popdist_rank == 0):
if not os.path.exists(opts.checkpoint_path):
os.makedirs(opts.checkpoint_path)
filename = f"{opts.model}_{opts.data}_{epoch}.pt"
save_path = os.path.join(opts.checkpoint_path, filename)
state = training_model.model.model.state_dict()
optimizer_state = optimizer.state_dict()
torch.save(
{
'epoch': epoch,
'model_state_dict': state,
'optimizer_state_dict': optimizer_state,
'loss': aggregated_loss,
'train_accuracy': aggregated_acc,
'opts': opts
}, save_path)
def train(
train_data,
val_data,
user_list_train_filtered,
user_list_val_filtered,
surr,
eta=0.1,
momentum=0.0,
lamb=0.1,
num_iter_val=5,
tolerance=1e-6,
num_total_iter_training=151,
draw=False,
verbose=True,
random_seed = 786,
w=None):
np.random.seed(random_seed)
metrics = utils.Metrics()
metrics.eta_lr = eta
metrics.lamb_reg = lamb
if w is None:
w = np.random.normal(0, 1, (train_data.shape[1] - 2, ))
prev_grad_w = np.zeros((train_data.shape[1] - 2, ))
for num_iter in np.arange(num_total_iter_training):
tic = time.time()
metrics.w_list.append(w)
loss_opt = 0
k_minus_w_opt = 0
grad_w = np.zeros((train_data.shape[1] - 2, ))
sorting_time = 0
surrogate_time = 0
for user_id in user_list_train_filtered:
user_df_train = train_data[train_data['user'] == user_id]
if(len(user_df_train.label.unique()) == 1):
if(user_df_train.label.iloc[0] == 1.0):
loss_opt += 0.0
else:
loss_opt += 1.0
else:
sorting_start_time = time.time()
beta = int(user_beta_train[user_id])
user_df_pos = user_df_train[user_df_train['label'] == 1]
user_df_neg = user_df_train[user_df_train['label'] == 0]
user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values
user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values
indices_pos, scores_pos = sort_order(user_feat_pos, w)
indices_neg, scores_neg = sort_order(user_feat_neg, w)
sorted_user_feat_pos = user_feat_pos[indices_pos, :]
sorted_user_feat_neg = user_feat_neg[indices_neg, :]
sorted_scores_pos = scores_pos[indices_pos]
sorted_scores_neg = scores_neg[indices_neg]
sorting_time += time.time() - sorting_start_time
surrogate_start_time = time.time()
pi_opt, score_mat = surr.compute_pi(
sorted_scores_pos, sorted_scores_neg,
w, k, beta)
loss_opt_user, _, _, _ = surr.loss(
pi_opt, sorted_scores_pos, sorted_scores_neg, k, beta)
if draw and user_id == 0:
plt.subplot(1,2,1)
plt.imshow(score_mat)
plt.subplot(1,2,2)
plt.imshow(pi_opt)
plt.show()
grad_w_user = surr.gradient(
sorted_user_feat_pos, sorted_user_feat_neg, pi_opt, k, beta)
surrogate_time += time.time() - surrogate_start_time
grad_w_user += lamb*w
loss_opt += loss_opt_user
grad_w += grad_w_user
grad_w = grad_w/len(user_list_train_filtered)
metrics_start_time = time.time()
# sort data once for both micro
user_feat = train_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(train_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.grad_w_list.append(np.linalg.norm(grad_w))
metrics.loss_opt_list_train.append(loss_opt/len(user_list_train_filtered))
metrics.micro_auc_rel_k_list_train.append(utils.compute_micro(data_true, user_list_train_filtered, user_beta_train, w, k))
if verbose:
print(' sorting elapsed time: ', sorting_time)
print(' surrogate elapsed time: ', surrogate_time)
print(' metrics elapsed time: ', time.time() - metrics_start_time)
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'loss train:',metrics.loss_opt_list_train[-1])
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'grad_w:',metrics.grad_w_list[-1])
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'microaucrelk train:',metrics.micro_auc_rel_k_list_train[-1])
else:
print('epoch', num_iter+1, 'completed. micro:{}'.format(metrics.micro_auc_rel_k_list_train[-1]))
if(num_iter%num_iter_val == 0):
loss_opt_val = 0
k_minus_w_opt_val = 0
for user_id in user_list_val_filtered:
user_df_val = val_data[val_data['user'] == user_id]
if(len(user_df_val.label.unique()) == 1):
if(user_df_val.label.iloc[0] == 1.0):
loss_opt_val += 0.0
else:
loss_opt_val += 1.0
else:
beta = int(user_beta_val[user_id])
user_df_pos = user_df_val[user_df_val['label'] == 1]
user_df_neg = user_df_val[user_df_val['label'] == 0]
user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values
user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values
indices_pos, scores_pos = sort_order(user_feat_pos, w)
indices_neg, scores_neg = sort_order(user_feat_neg, w)
sorted_user_feat_pos = user_feat_pos[indices_pos, :]
sorted_user_feat_neg = user_feat_neg[indices_neg, :]
sorted_scores_pos = scores_pos[indices_pos]
sorted_scores_neg = scores_neg[indices_neg]
pi_opt_val, score_mat_val = surr.compute_pi(sorted_scores_pos, sorted_scores_neg, w, k, beta)
loss_opt_user_val, _, _, _ = surr.loss(
pi_opt_val, sorted_scores_pos, sorted_scores_neg, k, beta)
if draw and user_id == 0:
plt.subplot(1,2,1)
plt.imshow(score_mat_val)
plt.subplot(1,2,2)
plt.imshow(pi_opt_val)
plt.show()
loss_opt_val += loss_opt_user_val
# sort data once for both micro
user_feat = val_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(val_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.loss_opt_list_val.append(loss_opt_val/len(user_list_val_filtered))
metrics.micro_auc_rel_k_list_val.append(utils.compute_micro(data_true, user_list_val_filtered, user_beta_val, w, k))
if verbose:
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'loss val:',metrics.loss_opt_list_val[-1])
print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'microaucrelk val:',metrics.micro_auc_rel_k_list_val[-1])
else:
print(' val micro:{}'.format(metrics.micro_auc_rel_k_list_val[-1]))
prev_grad_w = momentum * prev_grad_w + (1-momentum) * grad_w
w = w - (eta/np.sqrt(num_iter+1))*(prev_grad_w)
if verbose:
print('Epoch', num_iter+1, ' time taken is: ', time.time() - tic)
print("\n")
# also break if reached tolerance condition
if num_iter >= 10 and max(metrics.loss_opt_list_train[-10:])-min(metrics.loss_opt_list_train[-10:]) <= tolerance:
break
best_iter = (np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]//num_iter_val)*num_iter_val
best_iter = (np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]//num_iter_val)*num_iter_val
best_microaucrelk = metrics.micro_auc_rel_k_list_val[best_iter//num_iter_val]
print('Best micro aucrelk at iter: %d (metric: %f)' % (best_iter, best_microaucrelk))
return metrics, w
def train(
train_data,
val_data,
user_list_train_filtered,
user_list_val_filtered,
surr,
eta=0.1,
momentum=0.0,
lamb=0.1,
num_iter_val=5,
tolerance=1e-4,
num_total_iter_training=21,
draw=False,
verbose=True,
random_seed = 786,
w=None):
np.random.seed(random_seed)
metrics = utils.Metrics()
metrics.eta_lr = eta
metrics.lamb_reg = lamb
if w is None:
w = np.random.normal(0, 1, (train_data.shape[1] - 2, ))
prev_grad_w = np.zeros((train_data.shape[1] - 2, ))
for num_iter in np.arange(num_total_iter_training):
tic = time.time()
metrics.w_list.append(w)
loss_opt = 0
k_minus_w_opt = 0
grad_w = np.zeros((train_data.shape[1] - 2, ))
sorting_time = 0
surrogate_time = 0
pi_opt_avg_user_b = 0
pi_opt_avg_user_n = 0
for user_id in user_list_train_filtered:
user_df_train = train_data[train_data['user'] == user_id]
if(len(user_df_train.label.unique()) == 1):
if(user_df_train.label.iloc[0] == 1.0):
loss_opt += 0.0
else:
loss_opt += 1.0
else:
sorting_start_time = time.time()
beta = int(user_beta_train[user_id])
user_df_pos = user_df_train[user_df_train['label'] == 1]
user_df_neg = user_df_train[user_df_train['label'] == 0]
user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values
user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values
indices_pos, scores_pos = sort_order(user_feat_pos, w)
indices_neg, scores_neg = sort_order(user_feat_neg, w)
sorted_user_feat_pos = user_feat_pos[indices_pos, :]
sorted_user_feat_neg = user_feat_neg[indices_neg, :]
sorted_scores_pos = scores_pos[indices_pos]
sorted_scores_neg = scores_neg[indices_neg]
sorting_time += time.time() - sorting_start_time
surrogate_start_time = time.time()
pi_opt, score_mat = surr.compute_pi(
sorted_scores_pos, sorted_scores_neg,
w, k, beta)
pi_opt_avg_user_b += np.sum(pi_opt)/(beta*k)
pi_opt_avg_user_n += np.sum(pi_opt)/(len(indices_pos)*k)
loss_opt_user, _, _, _ = surr.loss(
pi_opt, sorted_scores_pos, sorted_scores_neg, k, beta)
if draw and user_id == 0:
plt.subplot(1,2,1)
plt.imshow(score_mat)
plt.subplot(1,2,2)
plt.imshow(pi_opt)
plt.show()
grad_w_user = surr.gradient(
sorted_user_feat_pos, sorted_user_feat_neg, pi_opt, k, beta)
surrogate_time += time.time() - surrogate_start_time
grad_w_user += lamb*w
loss_opt += loss_opt_user
grad_w += grad_w_user
grad_w = grad_w/len(user_list_train_filtered)
pi_opt_avg_user_b = pi_opt_avg_user_b/len(user_list_train_filtered)
pi_opt_avg_user_n = pi_opt_avg_user_n/len(user_list_train_filtered)
metrics_start_time = time.time()
# sort data once for both micro
user_feat = train_data.drop(['user', 'label'], axis = 1).values
y_scores = user_feat.dot(w)
data_true = deepcopy(train_data)
data_true['scores'] = y_scores
data_true = data_true.sort_values(by='scores', ascending=False)
data_true = data_true.reset_index(drop=True)
metrics.grad_w_list.append(np.linalg.norm(grad_w))
metrics.loss_opt_list_train.append(loss_opt/len(user_list_train_filtered))
metrics.micro_auc_rel_k_list_train.append(utils.compute_micro(data_true, user_list_train_filtered, user_beta_train, w, k))
if verbose:
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'loss train:',metrics.loss_opt_list_train[-1])
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'grad_w:',metrics.grad_w_list[-1])
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'microaucrelk train:',metrics.micro_auc_rel_k_list_train[-1])
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'pi_opt_avg_user_b:',pi_opt_avg_user_b)
print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'pi_opt_avg_user_n:',pi_opt_avg_user_n)
else:
print('k=', k,'Epoch', num_iter+1, 'done. micro:{}'.format(metrics.micro_auc_rel_k_list_train[-1]))
prev_grad_w = momentum * prev_grad_w + (1-momentum) * grad_w
w = w - (eta/np.sqrt(num_iter+1))*(prev_grad_w)
if verbose:
print('Epoch', num_iter+1, ' time taken is: ', time.time() - tic)
print("\n")
# also break if reached tolerance condition
if num_iter >= 10 and max(metrics.loss_opt_list_train[-10:])-min(metrics.loss_opt_list_train[-10:]) <= tolerance:
break
# save output to file
best_iter = np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]
with open('../results/' + dataset + '/test_results/result-testcomp-{}-{}-{}-{}-{}.json'.format(eta, lamb, k, surr.name, random_seed), 'w') as fp:
json.dump(metrics.to_dict(best_iter), fp)
return metrics, w
def evaluate(test_tag_lists, pred_tag_lists, remove_O=False):
metrics = utils.Metrics(test_tag_lists, pred_tag_lists, remove_O=remove_O)
metrics.report_scores()
metrics.report_confusion_matrix()
def train(training_model,
training_data,
args,
lr_scheduler,
epochs,
optimizer,
validation_function=None):
logging.info("Training the model")
# A generic container used by the train function to set and update the host-side training state.
class TrainingState():
pass
state = TrainingState()
state.iterations_per_epoch = len(training_data)
metrics = utils.Metrics(running_mean_length=state.iterations_per_epoch,
distributed=args.use_popdist)
state.old_lr = lr_scheduler.get_last_lr()[0]
state.num_instances = args.popdist_size if args.use_popdist else 1
# Determine the loss scaling change points.
num_loss_scaling_steps = int(
math.log2(args.loss_scaling // args.initial_loss_scaling)) + 1
loss_scaling_steps = {
i * (args.epoch // num_loss_scaling_steps) + 1:
args.initial_loss_scaling * (2**i)
for i in range(num_loss_scaling_steps)
}
state.new_loss_scaling = state.old_loss_scaling = args.initial_loss_scaling
if args.mixup_enabled or args.cutmix_enabled:
augmentation_generator = np.random.default_rng(args.seed)
for state.epoch in epochs:
state.epoch_start_time = time.perf_counter()
state.epoch_sample_size = 0
logging.info(
f"Epoch {state.epoch}/{args.epoch + args.fine_tune_epoch}")
bar = tqdm(training_data, total=state.iterations_per_epoch)
if state.epoch in loss_scaling_steps.keys():
state.new_loss_scaling = loss_scaling_steps[state.epoch]
# Beginning of the epoch.
for state.batch_idx, (input_data, labels) in enumerate(bar):
state.epoch_progress = (state.epoch - 1) + (
state.batch_idx + 1) / state.iterations_per_epoch
state.epoch_sample_size += labels.size()[0]
if args.mixup_enabled or args.cutmix_enabled:
input_data = get_augmented_samples(args, input_data,
augmentation_generator)
if args.compile_only:
_ = training_model.compile(input_data, labels)
logging.info(
"Graph compilation complete, --compile-only was set, exiting."
)
sys.exit(0)
accuracy, loss, sublosses = training_model(input_data, labels)
if args.profile:
# Profile report is only generated for one iteration.
sys.exit(0)
running_mean_loss, running_mean_acc = handle_metrics(
metrics,
loss,
sublosses,
accuracy,
state,
bar,
validation_function,
training_model,
args,
)
update_lr(lr_scheduler, optimizer, training_model, state, args)
# End of the epoch.
persist_checkpoint(training_model, optimizer, state, running_mean_loss,
running_mean_acc, args)
def build_graph_train(self, x=None):
with tf.variable_scope('encoder') as scope:
y, logdet, z = self.encoder(x)
self.y, self.logdet, self.z = y, logdet, z
logdet_pior_tmp = tf.zeros_like(logdet)
#################################################
""" Loss """
#################################################
#
# * Here simply the $-logp(x)$
tfd = tf.contrib.distributions
self.beta_ph = tf.placeholder(tf.float32, [])
y_flatten = tf.reshape(y, [c.BATCH_SIZE, -1])
z_flatten = tf.reshape(z, [c.BATCH_SIZE, -1])
prior_y = tfd.MultivariateNormalDiag(loc=tf.zeros_like(y_flatten),
scale_diag=self.beta_ph *
tf.ones_like(y_flatten))
prior_z = tfd.MultivariateNormalDiag(loc=tf.zeros_like(z_flatten),
scale_diag=self.beta_ph *
tf.ones_like(z_flatten))
log_prob_y = prior_y.log_prob(y_flatten)
log_prob_z = prior_z.log_prob(z_flatten)
# ### The MLE loss
loss = log_prob_y + log_prob_z + logdet
loss = -tf.reduce_mean(loss)
# ### The L2 regularization loss
print('... setting up L2 regularziation')
trainable_variables = tf.trainable_variables()
l2_reg = 0.00001
l2_loss = l2_reg * tf.add_n([
tf.nn.l2_loss(v) for v in tqdm(trainable_variables,
total=len(trainable_variables),
leave=False)
])
# ### Total loss -logp(x) + l2_loss
loss_per_pixel = loss / c.IMAGE_SIZE / c.IMAGE_SIZE
total_loss = l2_loss + loss_per_pixel
# it should be moved to main()
#sess.run(tf.global_variables_initializer())
#################################################
""" Trainer """
#################################################
self.lr_ph = tf.placeholder(tf.float32)
print('... setting up optimizer')
optimizer = tf.train.AdamOptimizer(self.lr_ph)
self.train_op = optimizer.minimize(total_loss)
# it should be moved to main()
# ## Initialize Actnorms using DDI
"""
sess.run(tf.global_variables_initializer())
nets.initialize_actnorms(
sess,
feed_dict_fn=lambda: {beta_ph: 1.0},
actnorm_layers=actnorm_layers,
num_steps=10,
)
"""
# ## Train model, define metrics and trainer
print('... setting up training metrics')
self.metrics = utils.Metrics(50,
metrics_tensors={
"total_loss": total_loss,
"loss_per_pixel": loss_per_pixel,
"l2_loss": l2_loss
})
self.plot_metrics_hook = utils.PlotMetricsHook(self.metrics, step=1000)
#################################################
""" Backward Flow """
#################################################
with tf.variable_scope('decoder') as scope:
self.x_reconst_train = self.decoder((y, logdet, z))
sample_y_flatten = prior_y.sample()
sample_y = tf.reshape(sample_y_flatten, y.shape.as_list())
sample_z = tf.reshape(prior_z.sample(), z.shape.as_list())
sampled_logdet = prior_y.log_prob(sample_y_flatten)
with tf.variable_scope(scope, reuse=True):
self.x_sampled_train = self.decoder(
(sample_y, sampled_logdet, sample_z))
return
