def evaluation(): args = parse() class_label_dict, label_class_dict = _read_label_file(args.label) classes = list() num_classes = len(class_label_dict) for i in range(num_classes): classes.append(label_class_dict[i]) gt_dict = utils.read_json(args.gt, class_label_dict) pred_dict = utils.read_log(args.log, class_label_dict) y_true, y_pred = utils.gen_yture_ypred(gt_dict, pred_dict) metrics = utils.Metrics(y_true, y_pred, classes) conf_matrix = metrics.confusion_matrix metrics_list = metrics.metrics_list() # print result print("class evaluation") print("~" * 50) print("accuracy: %.6f" % (metrics_list[-1])) print("~" * 50) for i in range(num_classes): print("%s_recall: %.6f" % (label_class_dict[i], metrics_list[i * 2])) print("%s_precision: %.6f" % (label_class_dict[i], metrics_list[i * 2 + 1])) print("~" * 50) print("Confusion Matrix") print(conf_matrix) if args.verbose: metrics.plot_confusion_matrix() print("Done.")
def evaluate(opt, dloader, model, epoch=0, vis=None, use_saved_file=False): # Visualizer opt.save_visuals = True if vis is None: if hasattr(opt, 'save_visuals') and opt.save_visuals: vis = Visualizer(os.path.join(opt.ckpt_path, 'test_log')) else: opt.save_visuals = False model.setup(is_train=False) metric = utils.Metrics() results = {} for step, data in enumerate(dloader): input, output, input_unocc, output_unocc = data dec_output, latent, nelbo = model.test(input, output) # results with partial occlusion in the TOP: crop_size_1 = opt.crop_size[1] output_eval = torch.cat([input_unocc, output], dim=1)[:, :, :, -crop_size_1:] rec_pred_eval = dec_output[:, :, :, -crop_size_1:] metric.update(output_eval, rec_pred_eval) if (step + 1) % opt.log_every == 0: print('{}/{}'.format(step + 1, len(dloader))) if opt.save_visuals: vis.add_images(model.get_visuals(), step, prefix='test_val') # BCE, MSE results.update(metric.get_scores()) return results
def evaluate(opt, dloader, model, use_saved_file=False): # Visualizer if hasattr(opt, 'save_visuals') and opt.save_visuals: vis = Visualizer(os.path.join(opt.ckpt_path, 'tb_test')) else: opt.save_visuals = False model.setup(is_train=False) metric = utils.Metrics() results = {} if hasattr(opt, 'save_all_results') and opt.save_all_results: save_dir = os.path.join(opt.ckpt_path, 'results') os.makedirs(save_dir, exist_ok=True) else: opt.save_all_results = False # Hacky is_bouncing_balls = ('bouncing_balls' in opt.dset_name) and opt.n_components == 4 if is_bouncing_balls: dloader.dataset.return_positions = True saved_positions = os.path.join(opt.ckpt_path, 'positions.npy') if use_saved_file else '' velocity_metric = utils.VelocityMetrics(saved_positions) count = 0 for step, data in enumerate(dloader): if not is_bouncing_balls: input, gt = data else: input, gt, positions = data output, latent = model.test(input, gt) pred = output[:, opt.n_frames_input:, ...] metric.update(gt, pred) if opt.save_all_results: gt = np.concatenate([input.numpy(), gt.numpy()], axis=1) prediction = utils.to_numpy(output) count = save_images(prediction, gt, latent, save_dir, count) if is_bouncing_balls: # Calculate position and velocity from pose pose = latent['pose'].data.cpu() velocity_metric.update(positions, pose, opt.n_frames_input) if (step + 1) % opt.log_every == 0: print('{}/{}'.format(step + 1, len(dloader))) if opt.save_visuals: vis.add_images(model.get_visuals(), step, prefix='test') # BCE, MSE results.update(metric.get_scores()) if is_bouncing_balls: # Don't break the original code dloader.dataset.return_positions = False results.update(velocity_metric.get_scores()) return results
def evaluate(opt, dloader, model, use_saved_file=False): # Visualizer if hasattr(opt, 'save_visuals') and opt.save_visuals: vis = Visualizer(os.path.join(opt.ckpt_path, 'tb_test')) else: opt.save_visuals = False model.setup(is_train=False) metric = utils.Metrics() results = {} if hasattr(opt, 'save_all_results') and opt.save_all_results: save_dir = os.path.join(opt.ckpt_path, 'results') os.makedirs(save_dir, exist_ok=True) else: opt.save_all_results = False count = 0 results = [] for step, data in enumerate(dloader): input, output, neigh, dist, man = data res_dict = model.test(*data) results.append(res_dict) # if opt.save_all_results: # gt = np.concatenate([input.numpy(), gt.numpy()], axis=1) # prediction = utils.to_numpy(dec_output) # count = save_images(prediction, gt, latent) if (step + 1) % opt.log_every == 0: print('{}/{}'.format(step + 1, len(dloader))) # if opt.save_visuals: # vis.add_images(model.get_visuals(), step, prefix='test') final_results = { 'Rank_G': 0, 'Trace_K': 0, 'Local_geom': 0, 'Total_loss': 0 } N = len(results) for item in results: final_results['Rank_G'] += item['Rank_G'] / N final_results['Trace_K'] += item['Trace_K'] / N final_results['Local_geom'] += item['Local_geom'] / N final_results['Total_loss'] += item['Total_loss'] / N # MSE # results.update(metric.get_scores()) # TODO: metric.reset? return final_results
def evaluate(opt, model, data_loader, logger, error_threshold=0.05, limit=None, vis=None): ''' Loop through the dataset and calculate evaluation metrics. ''' if model.compare_model is not None: logger.print('Comparison: {} ({}), {} ({})'.format(\ model.iterator.name(), model.iterator.n_operations, model.compare_model.name(), model.compare_model.n_operations)) logger.print('Initialization: {}'.format(opt.initialization)) logger.print('Error threshold: {}'.format(error_threshold)) metric = utils.Metrics(scale=1, error_threshold=error_threshold) images = {'error_curves': [], 'results': []} for step, data in enumerate(data_loader): bc, gt, x = data['bc'], data['final'], data['x'] f = None if 'f' not in data else data['f'] if opt.initialization != 'random': # Test time: do not change data if 'random' x = utils.initialize(x, bc, opt.initialization) results, x = model.evaluate(x, gt, bc, f, opt.n_evaluation_steps) # Update metric metric.update(results) if step % opt.log_every == 0: img = utils.plot_error_curves(results, num=4) if vis is not None: vis.add_image({'errors_avg_init': img}, step) images['error_curves'].append(img) img = utils.plot_results({'x': x, 'gt': gt}) if vis is not None: vis.add_image({'results': img}, step) images['results'].append(img) if (step + 1) % opt.log_every == 0: print('Step {}'.format(step + 1)) if limit is not None and (step + 1) == limit: break # Get results results = metric.get_results() for key in results: logger.print('{}: {}'.format(key, results[key])) metric.reset() return results, images
def main(): args = argparser() # json 格式的 ground truth gt_dict = utils.read_json(args.gt) # gt_dict = utils.read_url_list(args.gt) infered_dict_list = infer.infer(gt_dict, args.tool, ak=args.ak, sk=args.sk) if args.log: utils.logs(infered_dict_list, args.log) y_true, y_pred = utils.get_true_pred(gt_dict, infered_dict_list) metric = utils.Metrics(y_true, y_pred) conf_matrix = metric.confusion_matrix() acc = metric.accuracy() pulp_recall = metric.pulp_recall() pulp_precision = metric.pulp_precision() sexy_recall = metric.sexy_recall() sexy_precision = metric.sexy_precision() normal_recall = metric.normal_recall() normal_precision = metric.normal_precision() print('\n') print('【%s】剑皇测试' % __NAME[args.tool]) print('~' * 50) print('Ground Truth: ') print('总样本: %d' % len(gt_dict)) print('有效识别样本: %d' % np.sum(conf_matrix)) print('~' * 50) print('测试集分布: ') print('%d 个色情样本' % (np.sum(conf_matrix, axis=1)[0])) print('%d 个性感样本' % (np.sum(conf_matrix, axis=1)[1])) print('%d 个正常样本' % (np.sum(conf_matrix, axis=1)[2])) print('~' * 50) print('模型指标: ') print('accuracy: %f ' % acc) print('pulp_recall: %f ' % pulp_recall) print('pulp_precision: %f ' % pulp_precision) print('sexy_recall: %f ' % sexy_recall) print('sexy_precision: %f ' % sexy_precision) print('normal_recall: %f ' % normal_recall) print('normal_precision: %f ' % normal_precision) print('~' * 50) print('Confusion Matrix: ') print(conf_matrix) print('\n') if args.vis: metric.plot_confusion_matrix()
def decision_tree_top3_feats_predict_result(): """ 用所有次的 ['乳酸脱氢酶', '超敏C反应蛋白', '淋巴细胞(%)'] 进行预测 孙川 2020.04.07 """ # 决定是否使用每个病人最后一天的数据 last_sample = False data1 = pd.read_parquet('data/time_series_375.parquet')[[ '乳酸脱氢酶', '超敏C反应蛋白', '淋巴细胞(%)', '出院时间', '出院方式' ]] data2 = pd.read_parquet('data/time_series_test_110.parquet')[[ '乳酸脱氢酶', '超敏C反应蛋白', '淋巴细胞(%)', '出院时间', '出院方式' ]] # data1是375 data2是110 concat是485 for data in [data1, data2, utils.concat_data(data1, data2)]: # 滑窗合并数据 data = utils.merge_data_by_sliding_window(data, n_days=1, dropna=True, subset=utils.top3_feats_cols, time_form='diff') #是否使用最后一次的数据,因为merge_data_by_sliding_window中last自带升序因此这里取first() #groupby后,每个病人的索引是二级索引t_diff,是升序 if last_sample: data = data.groupby('PATIENT_ID').first() # 论文决策树预测 data['pred'] = data.apply(utils.decision_tree, axis=1) # 调用自己写的结果统计方式utils.Metrics metrics = utils.Metrics(acc='overall', f1='overall', conf_mat='overall', report='overall') metrics.record(data['出院方式'], data['pred']) metrics.print_metrics()
def main(): ## get model/training params args = parser.parse_args() if args.debug: print ('==== DEBUGGING MODE ====') # get name of script for saving models script_name = os.path.basename(__file__) ## Initialize metrics ### TrainingEval = utils.TrainingMetrics(script_name) working_dir = TrainingEval.working_dir valid = Prepare_Data(args.data,'valid/valid') valid_batches = DataLoader(valid, args.batch_size, drop_last=True, shuffle=True) Validation = utils.Metrics(valid_batches, working_dir ,'validation') # cp running script to working dir. os.system('cp {} {}'.format(script_name, working_dir)) ## Initialize model if torch.cuda.is_available(): model = ConvNet(args.kernel_size, args.stride, args.padding, args.ks_pool, args.str_pool, args.pad_pool).cuda() else: model = ConvNet(args.kernel_size, args.stride, args.padding, args.ks_pool, args.str_pool, args.pad_pool) ## log model/training params to file LogFile = utils.LogFile(args, model, working_dir) ## Loss and optimizer criterion = nn.CrossEntropyLoss() # doees not ignore padding (0) ignore_index=0 optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) # Train the model step = -1 # nr of batches loss_list = [] acc_list = [] valid_loss_list = [] valid_acc_list = [] for epoch in range(args.num_epochs): for train_ds in range(0,10): f = args.data name = 'train/train_{}'.format(train_ds) train = Prepare_Data(f,name) train_batches = DataLoader(train, batch_size=args.batch_size, drop_last=True, shuffle=True) for i, batch in enumerate(train_batches): step += 1 # one hot encode batch = utils.to_one_hot(batch) # transpose to input seq as vector batch = torch.transpose(batch,1,2) #transpose dim 1,2 => channels=aa ## Run the forward pass ## out = model(batch) # sandsynligheder=> skal være [10,25,502] hvor de 25 er sandsynligheder # convert back to aa labels from one hot for loss batch_labels = utils.from_one_hot(batch) # integers for labels med 100% sikkerhed ## loss ## loss = criterion(out, batch_labels) loss_list.append(loss.item()) ## switch model to training mode, clear gradient accumulators ## model.train() optimizer.zero_grad() ## Backprop and perform Adam optimisation ## loss.backward() optimizer.step() ## Track the accuracy ## if i % 50 == 0: # ########## acc = TrainingEval.get_acc(out,batch_labels) acc_list.append(acc) TrainingEval.save_metrics(acc, loss.item(), step, epoch) print('Epoch [{}/{}], Step: {}, Loss: {:.4f}, Accuracy: {:.4f}%' .format(epoch + 1, args.num_epochs, step, loss.item(), acc*100)) # Validation ## if i % 1000 == 0: val_loss, val_acc, conf_matrix = \ Validation.get_performance(model,criterion, confusion_matrix = True) Validation.save(val_acc, val_loss, epoch, step) # add to list for fast plotting valid_loss_list.append(val_loss) valid_acc_list.append(val_acc) print('Validation: Loss: {:.4f}, Accuracy: {:.4f}%\n' .format(val_loss, val_acc*100)) # plot TrainingEval.plot_metrics(acc_list, loss_list, valid_acc_list, valid_loss_list, epoch) Validation.plot_confusion_matrix(conf_matrix) Validation.plot_per_class(conf_matrix) # if i % 2000 == 0: # # Save the model # TrainingEval.save_model(model.state_dict(), i) # LogFile.log_saved_model(step) # Save the model every two train_-ds if train_ds % 5 ==0: utils.save_checkpoint(model, optimizer, epoch, train_ds,loss_list, acc_list, working_dir) utils.save_final_model(model, working_dir) LogFile.log_saved_model(step) LogFile.log_performance(acc, loss.item(), ds_type='Training') if args.testing: f = args.data name = 'test/test_1' test = Prepare_Data(f,name) test_batches = DataLoader(test, batch_size=args.batch_size, drop_last=True, shuffle=True) Test = utils.Metrics(test_batches, working_dir ,'test') test_loss, test_acc, conf_matrix = Test.get_performance( model, criterion, confusion_matrix = True) Test.save(test_acc, test_loss, epoch=-1, step=-1) Test.plot_confusion_matrix(conf_matrix) Test.save_conf_matrix(conf_matrix) Test.plot_per_class(conf_matrix) LogFile.log_performance(test_acc, test_loss, ds_type='Test')
def train(training_model, training_data, opts, lr_scheduler, epochs, optimizer, validation_function=None): old_lr = lr_scheduler.get_last_lr()[0] iterations_per_epoch = len(training_data) num_instances = opts.popdist_size if opts.use_popdist else 1 metrics = utils.Metrics(running_mean_length=iterations_per_epoch, distributed=opts.use_popdist) # Determine loss scaling change points num_loss_scaling_steps = int( math.log2(opts.loss_scaling // opts.initial_loss_scaling)) + 1 loss_scaling_steps = { i * (opts.epoch // num_loss_scaling_steps) + 1: opts.initial_loss_scaling * (2**i) for i in range(num_loss_scaling_steps) } new_loss_scaling = old_loss_scaling = opts.initial_loss_scaling for epoch in epochs: logging.info(f"Epoch {epoch}/{opts.epoch}") if opts.disable_metrics: bar = training_data else: bar = tqdm(training_data, total=iterations_per_epoch) epoch_start_time = time.time() total_sample = 0 if epoch in loss_scaling_steps.keys(): new_loss_scaling = loss_scaling_steps[epoch] for batch_idx, (input_data, labels) in enumerate(bar): preds, losses = training_model(input_data, labels) epoch_num = epoch - 1 + float(batch_idx + 1) / iterations_per_epoch if not opts.disable_metrics: with torch.no_grad(): mean_loss = torch.mean(losses).item() acc = utils.accuracy(preds, labels) metrics.save_value("accuracy", acc) metrics.save_value("loss", mean_loss) aggregated_loss = metrics.get_running_mean("loss") aggregated_acc = metrics.get_running_mean("accuracy") bar.set_description( f"Loss:{aggregated_loss:0.4f} | Accuracy:{aggregated_acc:0.2f}%" ) total_sample += input_data.size()[0] * num_instances if not opts.disable_metrics and ( (batch_idx + 1) % (iterations_per_epoch // opts.logs_per_epoch) == 0): elapsed_time = metrics.get_elapsed_time() num_batches = metrics.get_count() if validation_function is not None and ( epoch % opts.validation_frequency == 0) and (not opts.use_popdist or opts.popdist_rank == 0): training_model.detachFromDevice() validation_accuracy = validation_function() model.train() training_model.attachToDevice() logging.info( f"Validation Accuracy: {validation_accuracy:0.2f}%") else: validation_accuracy = 0.0 # save metrics result_dict = { "loss_avg": metrics.get_running_mean("loss"), "loss_batch": metrics.get_value("loss"), "epoch": epoch_num, "iteration": batch_idx + 1 + (epoch - 1) * iterations_per_epoch, "train_accuracy_avg": metrics.get_running_mean("accuracy"), "train_accuracy_batch": metrics.get_value("accuracy"), "learning_rate": old_lr, "loss_scaling": old_loss_scaling, "train_img_per_sec": (num_batches * input_data.size()[0] / elapsed_time), "latency_sec": elapsed_time / (num_batches * num_instances), "validation_accuracy": validation_accuracy } utils.Logger.log_train_results(result_dict) # lr schedule lr_scheduler.step(epoch_num) new_lr = lr_scheduler.get_last_lr()[0] if new_lr != old_lr or new_loss_scaling != old_loss_scaling: if new_loss_scaling != old_loss_scaling: optimizer.loss_scaling = new_loss_scaling if opts.optimizer == 'sgd': optimizer.param_groups[0][ "velocity_scaling"] = new_loss_scaling / opts.loss_velocity_scaling_ratio optimizer.param_groups[1][ "velocity_scaling"] = new_loss_scaling / opts.loss_velocity_scaling_ratio training_model.setOptimizer(optimizer) old_lr = new_lr old_loss_scaling = new_loss_scaling if opts.lr_schedule == "step": logging.info(f"Learning rate is changed to {new_lr}") epoch_end_time = time.time() if not opts.disable_metrics: aggregated_acc = metrics.get_running_mean("accuracy") logging.info( f"Epoch {epoch}: Train accuracy is {aggregated_acc:0.2f}%") elapsed_time = epoch_end_time - epoch_start_time # sync metrics if opts.use_popdist: total_sample = utils.sync_metrics(total_sample) elapsed_time = utils.sync_metrics(elapsed_time) epoch_throughput = total_sample / elapsed_time logging.info( f"Throughput of the epoch:{epoch_throughput:0.1f} img/sec") # save, in case of multiple processes save the weights only from the first instance. if not opts.checkpoint_path == "" and (not opts.use_popdist or opts.popdist_rank == 0): if not os.path.exists(opts.checkpoint_path): os.makedirs(opts.checkpoint_path) filename = f"{opts.model}_{opts.data}_{epoch}.pt" save_path = os.path.join(opts.checkpoint_path, filename) state = training_model.model.model.state_dict() optimizer_state = optimizer.state_dict() torch.save( { 'epoch': epoch, 'model_state_dict': state, 'optimizer_state_dict': optimizer_state, 'loss': aggregated_loss, 'train_accuracy': aggregated_acc, 'opts': opts }, save_path)
def train( train_data, val_data, user_list_train_filtered, user_list_val_filtered, surr, eta=0.1, momentum=0.0, lamb=0.1, num_iter_val=5, tolerance=1e-6, num_total_iter_training=151, draw=False, verbose=True, random_seed = 786, w=None): np.random.seed(random_seed) metrics = utils.Metrics() metrics.eta_lr = eta metrics.lamb_reg = lamb if w is None: w = np.random.normal(0, 1, (train_data.shape[1] - 2, )) prev_grad_w = np.zeros((train_data.shape[1] - 2, )) for num_iter in np.arange(num_total_iter_training): tic = time.time() metrics.w_list.append(w) loss_opt = 0 k_minus_w_opt = 0 grad_w = np.zeros((train_data.shape[1] - 2, )) sorting_time = 0 surrogate_time = 0 for user_id in user_list_train_filtered: user_df_train = train_data[train_data['user'] == user_id] if(len(user_df_train.label.unique()) == 1): if(user_df_train.label.iloc[0] == 1.0): loss_opt += 0.0 else: loss_opt += 1.0 else: sorting_start_time = time.time() beta = int(user_beta_train[user_id]) user_df_pos = user_df_train[user_df_train['label'] == 1] user_df_neg = user_df_train[user_df_train['label'] == 0] user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values indices_pos, scores_pos = sort_order(user_feat_pos, w) indices_neg, scores_neg = sort_order(user_feat_neg, w) sorted_user_feat_pos = user_feat_pos[indices_pos, :] sorted_user_feat_neg = user_feat_neg[indices_neg, :] sorted_scores_pos = scores_pos[indices_pos] sorted_scores_neg = scores_neg[indices_neg] sorting_time += time.time() - sorting_start_time surrogate_start_time = time.time() pi_opt, score_mat = surr.compute_pi( sorted_scores_pos, sorted_scores_neg, w, k, beta) loss_opt_user, _, _, _ = surr.loss( pi_opt, sorted_scores_pos, sorted_scores_neg, k, beta) if draw and user_id == 0: plt.subplot(1,2,1) plt.imshow(score_mat) plt.subplot(1,2,2) plt.imshow(pi_opt) plt.show() grad_w_user = surr.gradient( sorted_user_feat_pos, sorted_user_feat_neg, pi_opt, k, beta) surrogate_time += time.time() - surrogate_start_time grad_w_user += lamb*w loss_opt += loss_opt_user grad_w += grad_w_user grad_w = grad_w/len(user_list_train_filtered) metrics_start_time = time.time() # sort data once for both micro user_feat = train_data.drop(['user', 'label'], axis = 1).values y_scores = user_feat.dot(w) data_true = deepcopy(train_data) data_true['scores'] = y_scores data_true = data_true.sort_values(by='scores', ascending=False) data_true = data_true.reset_index(drop=True) metrics.grad_w_list.append(np.linalg.norm(grad_w)) metrics.loss_opt_list_train.append(loss_opt/len(user_list_train_filtered)) metrics.micro_auc_rel_k_list_train.append(utils.compute_micro(data_true, user_list_train_filtered, user_beta_train, w, k)) if verbose: print(' sorting elapsed time: ', sorting_time) print(' surrogate elapsed time: ', surrogate_time) print(' metrics elapsed time: ', time.time() - metrics_start_time) print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'loss train:',metrics.loss_opt_list_train[-1]) print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'grad_w:',metrics.grad_w_list[-1]) print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'microaucrelk train:',metrics.micro_auc_rel_k_list_train[-1]) else: print('epoch', num_iter+1, 'completed. micro:{}'.format(metrics.micro_auc_rel_k_list_train[-1])) if(num_iter%num_iter_val == 0): loss_opt_val = 0 k_minus_w_opt_val = 0 for user_id in user_list_val_filtered: user_df_val = val_data[val_data['user'] == user_id] if(len(user_df_val.label.unique()) == 1): if(user_df_val.label.iloc[0] == 1.0): loss_opt_val += 0.0 else: loss_opt_val += 1.0 else: beta = int(user_beta_val[user_id]) user_df_pos = user_df_val[user_df_val['label'] == 1] user_df_neg = user_df_val[user_df_val['label'] == 0] user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values indices_pos, scores_pos = sort_order(user_feat_pos, w) indices_neg, scores_neg = sort_order(user_feat_neg, w) sorted_user_feat_pos = user_feat_pos[indices_pos, :] sorted_user_feat_neg = user_feat_neg[indices_neg, :] sorted_scores_pos = scores_pos[indices_pos] sorted_scores_neg = scores_neg[indices_neg] pi_opt_val, score_mat_val = surr.compute_pi(sorted_scores_pos, sorted_scores_neg, w, k, beta) loss_opt_user_val, _, _, _ = surr.loss( pi_opt_val, sorted_scores_pos, sorted_scores_neg, k, beta) if draw and user_id == 0: plt.subplot(1,2,1) plt.imshow(score_mat_val) plt.subplot(1,2,2) plt.imshow(pi_opt_val) plt.show() loss_opt_val += loss_opt_user_val # sort data once for both micro user_feat = val_data.drop(['user', 'label'], axis = 1).values y_scores = user_feat.dot(w) data_true = deepcopy(val_data) data_true['scores'] = y_scores data_true = data_true.sort_values(by='scores', ascending=False) data_true = data_true.reset_index(drop=True) metrics.loss_opt_list_val.append(loss_opt_val/len(user_list_val_filtered)) metrics.micro_auc_rel_k_list_val.append(utils.compute_micro(data_true, user_list_val_filtered, user_beta_val, w, k)) if verbose: print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'loss val:',metrics.loss_opt_list_val[-1]) print('Epoch', num_iter+1, 'completed out of',num_total_iter_training, 'for', surr.name, 'microaucrelk val:',metrics.micro_auc_rel_k_list_val[-1]) else: print(' val micro:{}'.format(metrics.micro_auc_rel_k_list_val[-1])) prev_grad_w = momentum * prev_grad_w + (1-momentum) * grad_w w = w - (eta/np.sqrt(num_iter+1))*(prev_grad_w) if verbose: print('Epoch', num_iter+1, ' time taken is: ', time.time() - tic) print("\n") # also break if reached tolerance condition if num_iter >= 10 and max(metrics.loss_opt_list_train[-10:])-min(metrics.loss_opt_list_train[-10:]) <= tolerance: break best_iter = (np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]//num_iter_val)*num_iter_val best_iter = (np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0]//num_iter_val)*num_iter_val best_microaucrelk = metrics.micro_auc_rel_k_list_val[best_iter//num_iter_val] print('Best micro aucrelk at iter: %d (metric: %f)' % (best_iter, best_microaucrelk)) return metrics, w
def train( train_data, val_data, user_list_train_filtered, user_list_val_filtered, surr, eta=0.1, momentum=0.0, lamb=0.1, num_iter_val=5, tolerance=1e-4, num_total_iter_training=21, draw=False, verbose=True, random_seed = 786, w=None): np.random.seed(random_seed) metrics = utils.Metrics() metrics.eta_lr = eta metrics.lamb_reg = lamb if w is None: w = np.random.normal(0, 1, (train_data.shape[1] - 2, )) prev_grad_w = np.zeros((train_data.shape[1] - 2, )) for num_iter in np.arange(num_total_iter_training): tic = time.time() metrics.w_list.append(w) loss_opt = 0 k_minus_w_opt = 0 grad_w = np.zeros((train_data.shape[1] - 2, )) sorting_time = 0 surrogate_time = 0 pi_opt_avg_user_b = 0 pi_opt_avg_user_n = 0 for user_id in user_list_train_filtered: user_df_train = train_data[train_data['user'] == user_id] if(len(user_df_train.label.unique()) == 1): if(user_df_train.label.iloc[0] == 1.0): loss_opt += 0.0 else: loss_opt += 1.0 else: sorting_start_time = time.time() beta = int(user_beta_train[user_id]) user_df_pos = user_df_train[user_df_train['label'] == 1] user_df_neg = user_df_train[user_df_train['label'] == 0] user_feat_pos = user_df_pos.drop(['user', 'label'], axis = 1).values user_feat_neg = user_df_neg.drop(['user', 'label'], axis = 1).values indices_pos, scores_pos = sort_order(user_feat_pos, w) indices_neg, scores_neg = sort_order(user_feat_neg, w) sorted_user_feat_pos = user_feat_pos[indices_pos, :] sorted_user_feat_neg = user_feat_neg[indices_neg, :] sorted_scores_pos = scores_pos[indices_pos] sorted_scores_neg = scores_neg[indices_neg] sorting_time += time.time() - sorting_start_time surrogate_start_time = time.time() pi_opt, score_mat = surr.compute_pi( sorted_scores_pos, sorted_scores_neg, w, k, beta) pi_opt_avg_user_b += np.sum(pi_opt)/(beta*k) pi_opt_avg_user_n += np.sum(pi_opt)/(len(indices_pos)*k) loss_opt_user, _, _, _ = surr.loss( pi_opt, sorted_scores_pos, sorted_scores_neg, k, beta) if draw and user_id == 0: plt.subplot(1,2,1) plt.imshow(score_mat) plt.subplot(1,2,2) plt.imshow(pi_opt) plt.show() grad_w_user = surr.gradient( sorted_user_feat_pos, sorted_user_feat_neg, pi_opt, k, beta) surrogate_time += time.time() - surrogate_start_time grad_w_user += lamb*w loss_opt += loss_opt_user grad_w += grad_w_user grad_w = grad_w/len(user_list_train_filtered) pi_opt_avg_user_b = pi_opt_avg_user_b/len(user_list_train_filtered) pi_opt_avg_user_n = pi_opt_avg_user_n/len(user_list_train_filtered) metrics_start_time = time.time() # sort data once for both micro user_feat = train_data.drop(['user', 'label'], axis = 1).values y_scores = user_feat.dot(w) data_true = deepcopy(train_data) data_true['scores'] = y_scores data_true = data_true.sort_values(by='scores', ascending=False) data_true = data_true.reset_index(drop=True) metrics.grad_w_list.append(np.linalg.norm(grad_w)) metrics.loss_opt_list_train.append(loss_opt/len(user_list_train_filtered)) metrics.micro_auc_rel_k_list_train.append(utils.compute_micro(data_true, user_list_train_filtered, user_beta_train, w, k)) if verbose: print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'loss train:',metrics.loss_opt_list_train[-1]) print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'grad_w:',metrics.grad_w_list[-1]) print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'microaucrelk train:',metrics.micro_auc_rel_k_list_train[-1]) print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'pi_opt_avg_user_b:',pi_opt_avg_user_b) print('k=', k,'Epoch', num_iter+1, 'done out of',num_total_iter_training, 'for', surr.name, 'pi_opt_avg_user_n:',pi_opt_avg_user_n) else: print('k=', k,'Epoch', num_iter+1, 'done. micro:{}'.format(metrics.micro_auc_rel_k_list_train[-1])) prev_grad_w = momentum * prev_grad_w + (1-momentum) * grad_w w = w - (eta/np.sqrt(num_iter+1))*(prev_grad_w) if verbose: print('Epoch', num_iter+1, ' time taken is: ', time.time() - tic) print("\n") # also break if reached tolerance condition if num_iter >= 10 and max(metrics.loss_opt_list_train[-10:])-min(metrics.loss_opt_list_train[-10:]) <= tolerance: break # save output to file best_iter = np.where(np.asarray(metrics.loss_opt_list_train)==np.min(metrics.loss_opt_list_train))[0][0] with open('../results/' + dataset + '/test_results/result-testcomp-{}-{}-{}-{}-{}.json'.format(eta, lamb, k, surr.name, random_seed), 'w') as fp: json.dump(metrics.to_dict(best_iter), fp) return metrics, w
def evaluate(test_tag_lists, pred_tag_lists, remove_O=False): metrics = utils.Metrics(test_tag_lists, pred_tag_lists, remove_O=remove_O) metrics.report_scores() metrics.report_confusion_matrix()
def train(training_model, training_data, args, lr_scheduler, epochs, optimizer, validation_function=None): logging.info("Training the model") # A generic container used by the train function to set and update the host-side training state. class TrainingState(): pass state = TrainingState() state.iterations_per_epoch = len(training_data) metrics = utils.Metrics(running_mean_length=state.iterations_per_epoch, distributed=args.use_popdist) state.old_lr = lr_scheduler.get_last_lr()[0] state.num_instances = args.popdist_size if args.use_popdist else 1 # Determine the loss scaling change points. num_loss_scaling_steps = int( math.log2(args.loss_scaling // args.initial_loss_scaling)) + 1 loss_scaling_steps = { i * (args.epoch // num_loss_scaling_steps) + 1: args.initial_loss_scaling * (2**i) for i in range(num_loss_scaling_steps) } state.new_loss_scaling = state.old_loss_scaling = args.initial_loss_scaling if args.mixup_enabled or args.cutmix_enabled: augmentation_generator = np.random.default_rng(args.seed) for state.epoch in epochs: state.epoch_start_time = time.perf_counter() state.epoch_sample_size = 0 logging.info( f"Epoch {state.epoch}/{args.epoch + args.fine_tune_epoch}") bar = tqdm(training_data, total=state.iterations_per_epoch) if state.epoch in loss_scaling_steps.keys(): state.new_loss_scaling = loss_scaling_steps[state.epoch] # Beginning of the epoch. for state.batch_idx, (input_data, labels) in enumerate(bar): state.epoch_progress = (state.epoch - 1) + ( state.batch_idx + 1) / state.iterations_per_epoch state.epoch_sample_size += labels.size()[0] if args.mixup_enabled or args.cutmix_enabled: input_data = get_augmented_samples(args, input_data, augmentation_generator) if args.compile_only: _ = training_model.compile(input_data, labels) logging.info( "Graph compilation complete, --compile-only was set, exiting." ) sys.exit(0) accuracy, loss, sublosses = training_model(input_data, labels) if args.profile: # Profile report is only generated for one iteration. sys.exit(0) running_mean_loss, running_mean_acc = handle_metrics( metrics, loss, sublosses, accuracy, state, bar, validation_function, training_model, args, ) update_lr(lr_scheduler, optimizer, training_model, state, args) # End of the epoch. persist_checkpoint(training_model, optimizer, state, running_mean_loss, running_mean_acc, args)
def build_graph_train(self, x=None): with tf.variable_scope('encoder') as scope: y, logdet, z = self.encoder(x) self.y, self.logdet, self.z = y, logdet, z logdet_pior_tmp = tf.zeros_like(logdet) ################################################# """ Loss """ ################################################# # # * Here simply the $-logp(x)$ tfd = tf.contrib.distributions self.beta_ph = tf.placeholder(tf.float32, []) y_flatten = tf.reshape(y, [c.BATCH_SIZE, -1]) z_flatten = tf.reshape(z, [c.BATCH_SIZE, -1]) prior_y = tfd.MultivariateNormalDiag(loc=tf.zeros_like(y_flatten), scale_diag=self.beta_ph * tf.ones_like(y_flatten)) prior_z = tfd.MultivariateNormalDiag(loc=tf.zeros_like(z_flatten), scale_diag=self.beta_ph * tf.ones_like(z_flatten)) log_prob_y = prior_y.log_prob(y_flatten) log_prob_z = prior_z.log_prob(z_flatten) # ### The MLE loss loss = log_prob_y + log_prob_z + logdet loss = -tf.reduce_mean(loss) # ### The L2 regularization loss print('... setting up L2 regularziation') trainable_variables = tf.trainable_variables() l2_reg = 0.00001 l2_loss = l2_reg * tf.add_n([ tf.nn.l2_loss(v) for v in tqdm(trainable_variables, total=len(trainable_variables), leave=False) ]) # ### Total loss -logp(x) + l2_loss loss_per_pixel = loss / c.IMAGE_SIZE / c.IMAGE_SIZE total_loss = l2_loss + loss_per_pixel # it should be moved to main() #sess.run(tf.global_variables_initializer()) ################################################# """ Trainer """ ################################################# self.lr_ph = tf.placeholder(tf.float32) print('... setting up optimizer') optimizer = tf.train.AdamOptimizer(self.lr_ph) self.train_op = optimizer.minimize(total_loss) # it should be moved to main() # ## Initialize Actnorms using DDI """ sess.run(tf.global_variables_initializer()) nets.initialize_actnorms( sess, feed_dict_fn=lambda: {beta_ph: 1.0}, actnorm_layers=actnorm_layers, num_steps=10, ) """ # ## Train model, define metrics and trainer print('... setting up training metrics') self.metrics = utils.Metrics(50, metrics_tensors={ "total_loss": total_loss, "loss_per_pixel": loss_per_pixel, "l2_loss": l2_loss }) self.plot_metrics_hook = utils.PlotMetricsHook(self.metrics, step=1000) ################################################# """ Backward Flow """ ################################################# with tf.variable_scope('decoder') as scope: self.x_reconst_train = self.decoder((y, logdet, z)) sample_y_flatten = prior_y.sample() sample_y = tf.reshape(sample_y_flatten, y.shape.as_list()) sample_z = tf.reshape(prior_z.sample(), z.shape.as_list()) sampled_logdet = prior_y.log_prob(sample_y_flatten) with tf.variable_scope(scope, reuse=True): self.x_sampled_train = self.decoder( (sample_y, sampled_logdet, sample_z)) return