def main(img1, img2, chg_map, args=None): img_shape = np.shape(img1) im1 = np.reshape(img1, newshape=[-1,img_shape[-1]]) im2 = np.reshape(img2, newshape=[-1,img_shape[-1]]) im1 = utils.normlize(im1) im2 = utils.normlize(im2) chg_ref = np.reshape(chg_map, newshape=[-1]) imm = None all_magnitude = None differ = np.zeros(shape=[np.shape(chg_ref)[0],net_shape[-1], args.iter]) # load cva pre-detection result ind = sio.loadmat(args.area+'/cva_ref.mat') cva_ind = ind['cva_ref'] cva_ind = np.reshape(cva_ind, newshape=[-1]) for k1 in range(args.iter): logging.info('In %2d-th iteration········' % (k1)) i1, i2 = utils.getTrainSamples(cva_ind, im1, im2, args.trn) loss_log, vpro, fcx, fcy, bval = dsfa( xtrain=i1, ytrain=i2, xtest=im1, ytest=im2, net_shape=net_shape, args=args) imm, magnitude, differ_map = utils.linear_sfa(fcx, fcy, vpro, shape=img_shape) magnitude = np.reshape(magnitude, img_shape[0:-1]) differ[:, :, k1] = differ_map if all_magnitude is None: all_magnitude = magnitude / np.max(magnitude) else: all_magnitude = all_magnitude + magnitude / np.max(magnitude) change_map = np.reshape(utils.kmeans(np.reshape(all_magnitude, [-1])), img_shape[0:-1]) logging.info('Max value of change magnitude: %.4f'%(np.max(all_magnitude))) logging.info('Min value of change magnitude: %.4f'%(np.min(all_magnitude))) # magnitude acc_un, acc_chg, acc_all2, acc_tp = utils.metric(1-change_map, chg_map) acc_un, acc_chg, acc_all3, acc_tp = utils.metric(change_map, chg_map) plt.imsave('results.png',all_magnitude, cmap='gray') #plt.show() return None
def train(train_queue, model, device, optimizer, criterion, args): objs = utils.AverageMeter() top1 = utils.AverageMeter() model.train() confusion_matrix = None for step, (input_train, target_train) in enumerate(train_queue): n = input_train.size(0) input_train = input_train.to(device) target_train = target_train.squeeze().long().to(device) logits = model(input_train) loss = criterion(logits, target_train) optimizer.zero_grad() loss.backward() optimizer.step() acc, confusion = utils.metric(logits, target_train) objs.update(loss.item(), n) top1.update(acc, n) if step % args.report_freq == 0: logging.info('Train | Step: %d | Loss: %e | Accuracy: %.3f', step, objs.avg, top1.avg) if confusion_matrix is None: confusion_matrix = confusion else: confusion_matrix += confusion return top1.avg, objs.avg, confusion_matrix
def __init__(self, target_img, source_tiles, tile_size, reuse_count=float('inf'), metric_type='color_distance', randomize=False, bw=False, color_scale=False): self.target_img = target_img self.reuse_count = reuse_count self.source_tiles = source_tiles self.tile_size = tile_size self.randomize = randomize self.target_tiles = [] self.bw = bw self.color_scale = color_scale self._get_target_tiles() self._process_source_tiles() shuffle(self.target_tiles) shuffle(self.source_tiles) self.tile_use_count = Counter(range(len(self.source_tiles))) if bw: self.target_tiles = convert_tile_to_bw(self.target_tiles) self.source_tiles = convert_tile_to_bw(self.source_tiles) self.metric = lambda img1, img2: metric(img1, img2, metric_type) print(len(self.target_tiles), len(self.source_tiles))
def train_by_seed(self, seed, module): seed_everything(seed) self.param["seed"] = seed self.param["model_param"]["random_state"] = seed kf_manager = KFManager(self.param, module) X = module["X"] y = module["y"] print(X.columns) oof_preds = np.zeros((X.shape[0], )) if self.param["metric"] == "RMSLE": y = np.log1p(y) for fold, (train_index, valid_index) in enumerate(kf_manager.split()): train_X = X.loc[train_index].drop("Publisher", axis=1).values valid_X = X.loc[valid_index].drop("Publisher", axis=1).values train_y = y.loc[train_index].values valid_y = y.loc[valid_index].values print(train_X.shape) print(valid_X.shape) oof = self.train_by_fold(train_X, train_y, valid_X, valid_y, self.param, fold) oof_preds[valid_index] += oof if self.param["metric"] == "RMSLE": oof_preds = np.expm1(oof_preds) y = np.expm1(y) err = metric(y, oof_preds) print(f"seed {seed} : RMSLE is {err}") return oof_preds, err
def evaluate(model, dataloader, device="cuda"): with torch.no_grad(): model.eval() final_loss = 0 counter = 0 final_outputs = [] final_targets = [] print(f"{'='*15}Evaluating{'='*15}") for _, data in tqdm(enumerate(dataloader), total=int(len(dataloader))): counter = counter + 1 image = data["image"] target = data["target"] image = image.to(device, dtype=torch.float) target = target.to(device, dtype=torch.float) outputs = model(image) loss = nn.BCEWithLogitsLoss()(outputs, target.view(-1, 1)) final_loss += loss final_outputs.append(outputs) final_targets.append(target) final_outputs = torch.cat(final_outputs).cpu().numpy() final_targets = torch.cat(final_targets).cpu().numpy() auc_metric = metric(final_targets, final_outputs) return final_loss / counter, auc_metric
def train(train_queue, valid_queue, model, device): objs = utils.AverageMeter() top1 = utils.AverageMeter() model.train() for step, (input_train, target_train) in enumerate(train_queue): n = input_train.size(0) # get a random minibatch from the search queue with replacement input_val, target_val = next(iter(valid_queue)) input_train = input_train.to(device) target_train = target_train.squeeze().long().to(device) input_val = input_val.to(device) target_val = target_val.squeeze().long().to(device) model.module.alphas_step(input_train, target_train, input_val, target_val, args.order) logits, loss = model.module.weights_step(input_train, target_train, args.grad_clip) acc = utils.metric(logits, target_train) objs.update(loss.item(), n) top1.update(acc, n) if step % args.report_freq == 0: logging.info('Train | Step: %d | Loss: %e | Accuracy: %.3f', step, objs.avg, top1.avg) return top1.avg, objs.avg
def model_predict(): os.environ["CUDA_VISIBLE_DEVICES"] = "0" if K.backend() == 'tensorflow': config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) tokenizer = get_tokenizer(args.bert_vocab_path) # read data and relation test_data, id2rel, rel2id, num_rels = load_data(args.test_path, args.rel_dict_path) # load model subject_model, object_model, hbt_model = E2EModel( args.bert_config_path, args.bert_checkpoint_path, args.LR, num_rels) hbt_model.load_weights(args.save_weights_path) isExactMatch = True if args.dataset == 'Wiki-KBP' else False if isExactMatch: print("Exact Match") else: print("Partial Match") precision, recall, f1_score = metric(subject_model, object_model, test_data, id2rel, tokenizer, isExactMatch, args.test_result_path) print(f'{precision}\t{recall}\t{f1_score}')
def dev(config, bert_config, dev_path, id2rel, tokenizer, output_path=None): dev_data = json.load(open(dev_path)) for sent in dev_data: data.to_tuple(sent) with torch.no_grad(): Bert_model = BertModel(bert_config).to(device).eval() submodel = sub_model(config).to(device).eval() objmodel = obj_model(config).to(device).eval() state = torch.load( os.path.join(config.output_dir, config.load_model_name)) Bert_model.load_state_dict(state['bert_state_dict']) submodel.load_state_dict(state['subject_state_dict']) objmodel.load_state_dict(state['object_state_dict']) precision, recall, f1 = utils.metric(Bert_model, submodel, objmodel, dev_data, id2rel, tokenizer, output_path=output_path) logger.info('precision: %.4f' % precision) logger.info('recall: %.4f' % recall) logger.info('F1: %.4f' % f1)
def test(self, epoch, iteration): # performance on the entire holdout set yhat_test = self.predict(self.holdout.X) corr_test = metric(yhat_test.ravel(), self.holdout.y) # performance on a subset of the training data training_sample_size = yhat_test.shape[0] inds = choice(self.training.y.shape[0], training_sample_size, replace=False) yhat_train = self.predict(self.training.X[inds, ...]) corr_train = metric(yhat_train.ravel(), self.training.y[inds]) # save the results to a CSV file results = [epoch, iteration, corr_train, corr_test] self.save_csv(results) return results
def eval(model, dataLoader_valid): with torch.no_grad(): model.eval() model.mode = 'valid' valid_loss, index_valid = 0, 0 all_results = [] all_masks = [] all_fcs = [] all_fc_labels = [] for valid_data in dataLoader_valid: images, masks = valid_data images = images.cuda() masks = masks.cuda() b = images.size(0) outs, fc = data_parallel(model, images) model.loss = model.get_loss(outs, fc, masks) outs = torch.sigmoid(outs).cpu() fc = torch.sigmoid(fc).cpu() masks = masks.cpu() # all_results.append(outs) # all_masks.append(masks) all_results.append(F.upsample_bilinear(outs, (256, 256))) all_masks.append((F.upsample_bilinear(masks, (256, 256)) > 0.5).float()) all_fcs.append(fc) all_fc_labels.append( (masks.view(b, -1).sum(-1) > 0).float().view(b, 1)) if index_valid == 0: drawing(images, outs, masks, mode='valid') b = len(masks) valid_loss += model.loss.item() * b index_valid += b valid_loss = valid_loss / index_valid all_results = torch.cat(all_results, 0) all_masks = torch.cat(all_masks, 0) ts = np.linspace(0.3, 0.7, 5) mIoUs = [] mnegIoUs = [] mposIoUs = [] for t in ts: s, s_neg, s_pos, _, _ = metric(all_results, all_masks, t) mIoUs.append(s) mnegIoUs.append(s_neg) mposIoUs.append(s_pos) max_iou = max(mIoUs) max_index = mIoUs.index(max_iou) best_t = ts[max_index] valid_iou = max_iou return valid_loss, valid_iou, best_t, mnegIoUs[max_index], mposIoUs[ max_index],
def evaluate_test_results(cur_predict, cur_target, class_predict, class_traget, saved_thresholds, epoch, start_time, eval_loss, stats_path): total_precision = [] total_recall = [] cur_precision, cur_recall = metric(class_predict, class_traget, saved_thresholds) total_precision.append(np.array(cur_precision)) total_recall.append(np.array(cur_recall)) TPVFs, dices, PPVs, FPVFs = segmentation_metrics(cur_predict, cur_target) logging.info( '***************************************************************************' ) logging.info( 'Esophagus --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] ' % (dices[0], TPVFs[0], PPVs[0])) logging.info( 'heart --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] ' % (dices[1], TPVFs[1], PPVs[1])) logging.info( 'trachea --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] ' % (dices[2], TPVFs[2], PPVs[2])) logging.info( 'aorta --> Global dice is [%.5f], TPR is [%.5f], Precision is [%.5f] ' % (dices[3], TPVFs[3], PPVs[3])) total_precision = np.stack(total_precision, 1) total_recall = np.stack(total_recall, 1) logging.info( 'Epoch[%d], [precision=%.4f, -->%.3f, -->%.3f, -->%.3f, -->%.3f], using %.1f s!' % (epoch, np.mean(total_precision), np.mean(total_precision[0]), np.mean(total_precision[1]), np.mean(total_precision[2]), np.mean(total_precision[3]), time.time() - start_time)) logging.info( 'Epoch[%d], [recall=%.4f, -->%.3f, -->%.3f, -->%.3f, -->%.3f], using %.1f s!' % (epoch, np.mean(total_recall), np.mean(total_recall[0]), np.mean(total_recall[1]), np.mean(total_recall[2]), np.mean(total_recall[3]), time.time() - start_time)) with open(stats_path, 'a') as f: writer = csv.writer(f) writer.writerow([ epoch, "NA", np.mean(eval_loss), np.mean(total_precision), np.mean(total_precision[0]), np.mean(total_precision[1]), np.mean(total_precision[2]), np.mean(total_precision[3]), np.mean(dices), dices[0], dices[1], dices[2], dices[3] ]) logging.info( 'Epoch[%d], [total loss=%.6f], mean_dice=%.4f, using %.1f s!' % (epoch, np.mean(eval_loss), np.mean(dices), time.time() - start_time)) logging.info( '***************************************************************************' ) return np.mean(dices), np.mean(total_precision)
def update(self, targets, outputs): probs = torch.sigmoid(outputs) dice, dice_neg, dice_pos, _, _ = metric(probs, targets, self.base_threshold) self.base_dice_scores.append(dice) self.dice_pos_scores.append(dice_pos) self.dice_neg_scores.append(dice_neg) preds = predict(probs, self.base_threshold) iou = compute_iou_batch(preds, targets, classes=[1]) self.iou_scores.append(iou)
def main(img1, img2, chg_map, args=None): img_shape = np.shape(img1) im1 = np.reshape(img1, newshape=[-1,img_shape[-1]]) im2 = np.reshape(img2, newshape=[-1,img_shape[-1]]) im1 = utils.normlize(im1) im2 = utils.normlize(im2) chg_ref = np.reshape(chg_map, newshape=[-1]) imm = None all_magnitude = None differ = np.zeros(shape=[np.shape(chg_ref)[0],net_shape[-1]]) # load cva pre-detection result ind = sio.loadmat(args.area+'/cva_ref.mat') cva_ind = ind['cva_ref'] cva_ind = np.reshape(cva_ind, newshape=[-1]) i1, i2 = utils.getTrainSamples(cva_ind, im1, im2, args.trn) loss_log, vpro, fcx, fcy, bval = dsfa( xtrain=i1, ytrain=i2, xtest=im1, ytest=im2, net_shape=net_shape, args=args) imm, magnitude, differ_map = utils.linear_sfa(fcx, fcy, vpro, shape=img_shape) magnitude = np.reshape(magnitude, img_shape[0:-1]) differ = differ_map change_map = np.reshape(utils.kmeans(np.reshape(magnitude, [-1])), img_shape[0:-1]) # magnitude acc_un, acc_chg, acc_all2, acc_tp = utils.metric(1-change_map, chg_map) acc_un, acc_chg, acc_all3, acc_tp = utils.metric(change_map, chg_map) plt.imsave('results.png',change_map, cmap='gray') #plt.show() return None
def __call__(self, module): train_preds = [] errs = [] # label_encode: for seed in self.param["seeds"]: oof_preds, err = self.train_by_seed(seed, module) train_preds.append(oof_preds) errs.append(err) train_preds = np.mean(np.array(train_preds), axis=0) err = metric(module["y"], train_preds) print(f"final RMSLE is {err}") module["train_preds"] = train_preds module["errs"] = errs module["err"] = err return module
def on_epoch_end(self, epoch, logs=None): precision, recall, f1 = metric(self.subject_model, self.object_model, self.eval_data, self.id2rel, self.tokenizer) if self.monitor_op(f1 - self.min_delta, self.best) or self.monitor_op( self.min_delta, f1): self.best = f1 self.wait = 0 self.model.save_weights(self.save_weights_path) else: self.wait += 1 if self.wait >= self.patience: self.stopped_epoch = epoch self.model.stop_training = True print('f1: %.4f, precision: %.4f, recall: %.4f, best f1: %.4f\n' % (f1, precision, recall, self.best))
def evaluate(test_queue, model, device): objs = utils.AverageMeter() top1 = utils.AverageMeter() for step, (input_test, target_test) in enumerate(test_queue): model.eval() input_test = input_test.to(device) target_test = target_test.squeeze().long().to(device) with torch.no_grad(): logits, loss = model.module.loss(input_test, target_test) acc = utils.metric(logits, target_test) n = input_test.size(0) objs.update(loss.item(), n) top1.update(acc, n) return top1.avg, objs.avg
def do_valid(net, valid_loader, out_dir=None): valid_loss = np.zeros(6, np.float32) valid_num = np.zeros_like(valid_loss) valid_probability = [[],[],[],] valid_truth = [[],[],[],] for t, (input, truth, infor) in enumerate(valid_loader): batch_size = len(infor) net.eval() input = input.cuda() truth = [t.cuda() for t in truth] with torch.no_grad(): logit = net(input) probability = logit_to_probability(logit) loss = criterion(logit, truth) correct = metric(probability, truth) loss = [l.item() for l in loss] l = np.array([ *loss, *correct, ])*batch_size n = np.array([ 1, 1, 1, 1, 1, 1 ])*batch_size valid_loss += l valid_num += n for i in range(NUM_TASK): valid_probability[i].append(probability[i].data.cpu().numpy()) valid_truth[i].append(truth[i].data.cpu().numpy()) print('\r %8d /%d'%(valid_num[0], len(valid_loader.dataset)),end='',flush=True) pass assert(valid_num[0] == len(valid_loader.dataset)) valid_loss = valid_loss/(valid_num+1e-8) for i in range(NUM_TASK): valid_probability[i] = np.concatenate(valid_probability[i]) valid_truth[i] = np.concatenate(valid_truth[i]) recall, avgerage_recall = compute_kaggle_metric(valid_probability, valid_truth) return valid_loss, (recall, avgerage_recall)
def train(data_loader, net, loss, epoch, optimizer, get_lr, save_dir): start_time = time.time() net.train() lr = get_lr(epoch) for param_group in optimizer.param_groups: param_group['lr'] = lr total_train_loss = [] class_predict = [] class_target = [] for i, sample in enumerate(data_loader): data = sample['image'] target_c = sample['label_c'] target_s = sample['label_s'] data = data.to(DEVICE) target_c = target_c.to(DEVICE) target_s = target_s.to(DEVICE) output_s, output_c = net(data) optimizer.zero_grad() cur_loss, _, _, c_p = loss(output_s, output_c, target_s, target_c) total_train_loss.append(cur_loss.item()) class_target.append(target_c.detach().cpu().numpy()) class_predict.append(c_p.detach().cpu().numpy()) cur_loss.backward() optimizer.step() logging.info( 'Epoch[%d], Batch [%d], total loss is %.6f, using %.1f s!' % (epoch, i, np.mean(total_train_loss), time.time() - start_time)) total_train_class_predict = np.concatenate(class_predict, 0) total_train_class_target = np.concatenate(class_target, 0) adaptive_thresholds = get_threshold(total_train_class_predict, total_train_class_target, 0.995) cur_precision, _ = metric(total_train_class_predict, total_train_class_target, adaptive_thresholds) logging.info( 'Epoch[%d], [precision=%.4f, -->%.3f, -->%.3f, -->%.3f, -->%.3f]' % (epoch, np.mean(cur_precision), np.mean(cur_precision[0]), np.mean(cur_precision[1]), np.mean( cur_precision[2]), np.mean(cur_precision[3]))) logging.info('the adaptive thresholds is [%.4f, %.4f, %.4f, %.4f]' % (adaptive_thresholds[0], adaptive_thresholds[1], adaptive_thresholds[2], adaptive_thresholds[3])) return np.mean(total_train_loss), adaptive_thresholds
def train(model, dataloader, optimizer, scaler, device="cuda"): model.train() final_loss = 0 counter = 0 final_outputs = [] final_targets = [] print(f"{'='*15}Training{'='*15}") for _, data in tqdm(enumerate(dataloader), total=int(len(dataloader))): counter = counter + 1 image = data["image"] targets = data["target"] image = image.to(device, dtype=torch.float) targets = targets.to(device, dtype=torch.float) optimizer.zero_grad() with amp.autocast(): outputs = model(image) loss = nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1)) scaler.scale(loss).backward() scaler.step(optimizer) scaler.update() final_loss += loss final_outputs.append(outputs) final_targets.append(targets) final_outputs = torch.cat(final_outputs).detach().cpu().numpy() final_targets = torch.cat(final_targets).detach().cpu().numpy() auc_metric = metric( final_targets, final_outputs) return final_loss / counter, auc_metric
def test(test_dataloader, test_entity_list, id2tag, model): """ 测试模型 """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.eval() output_tag_ids = [] with torch.no_grad(): for _, (batch_tensor, batch_sent_len, batch_tags) in enumerate(test_dataloader): batch_tensor = batch_tensor.to(device) batch_sent_len = batch_sent_len.to(device) batch_tags = batch_tags.to(device) tag_seq = model.sequence_decode(batch_tensor, batch_sent_len) output_tag_ids = output_tag_ids + tag_seq output_tag = entity_recover(output_tag_ids, id2tag) P, R, F1 = metric(output_tag, test_entity_list) return output_tag_ids, P, R, F1
def train(train_queue, model, device): objs = utils.AverageMeter() top1 = utils.AverageMeter() for step, (input_train, target_train) in enumerate(train_queue): model.train() n = input_train.size(0) input_train = input_train.to(device) target_train = target_train.squeeze().long().to(device) logits, loss = model.module.weights_step(input_train, target_train, args.grad_clip) acc = utils.metric(logits, target_train) objs.update(loss.item(), n) top1.update(acc, n) if step % args.report_freq == 0: logging.info('Train | Step: %d | Loss: %e | Accuracy: %.3f', step, objs.avg, top1.avg) return top1.avg, objs.avg
def evaluate(test_queue, model, device, criterion): objs = utils.AverageMeter() top1 = utils.AverageMeter() model.eval() confusion_matrix = None for step, (input_test, target_test) in enumerate(test_queue): input_test = input_test.to(device) target_test = target_test.squeeze().long().to(device) with torch.no_grad(): logits = model(input_test) loss = criterion(logits, target_test) acc, confusion = utils.metric(logits, target_test) n = input_test.size(0) objs.update(loss.item(), n) top1.update(acc, n) if confusion_matrix is None: confusion_matrix = confusion else: confusion_matrix += confusion return top1.avg, objs.avg, confusion_matrix
results = [] # Try to understand what the criterion means, why is it mse? # https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html neigh = KNeighborsClassifier(n_neighbors=10) neigh.fit(x_train, y_train_class) result = neigh.predict(x_test) import numpy as np # Write a function to describe the error from utils import metric error = metric(y_test, result) plt.close() print(error) i1, i2 = 0, 0 error = np.zeros(10) N = 100 #dauert lang !! while i1 < 10: #print(i1) while i2 < N: np.random.seed(i2 + 13 * i1 + 5) neigh = KNeighborsClassifier(n_neighbors=i1 + 1) neigh.fit(x_train, y_train_class) result = neigh.predict(x_test) error[i1] = error[i1] + metric(y_test, result) i2 = i2 + 1
def training(split_method, weight_grapheme_root, weight_vowel_diacritic, weight_consonant_diacritic, weight_grapheme, n_splits, fold, train_data_loader, val_data_loader, model_type, optimizer_name, lr_scheduler_name, lr, warmup_proportion, batch_size, valid_batch_size, num_epoch, start_epoch, accumulation_steps, checkpoint_folder, load_pretrain, seed, loss_type, early_stopping, beta, alpha, cutmix_prob, mixup_prob, apex, freeze_first, freeze_epoches): torch.cuda.empty_cache() COMMON_STRING = '@%s: \n' % os.path.basename(__file__) COMMON_STRING += '\tset random seed\n' COMMON_STRING += '\t\tseed = %d\n' % seed COMMON_STRING += '\tset cuda environment\n' COMMON_STRING += '\t\ttorch.__version__ = %s\n' % torch.__version__ COMMON_STRING += '\t\ttorch.version.cuda = %s\n' % torch.version.cuda COMMON_STRING += '\t\ttorch.backends.cudnn.version() = %s\n' % torch.backends.cudnn.version( ) try: COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = %s\n' % os.environ[ 'CUDA_VISIBLE_DEVICES'] NUM_CUDA_DEVICES = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) except Exception: COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = None\n' NUM_CUDA_DEVICES = 1 COMMON_STRING += '\t\ttorch.cuda.device_count() = %d\n' % torch.cuda.device_count( ) COMMON_STRING += '\n' checkpoint_folder = os.path.join(checkpoint_folder, model_type + '/' + loss_type + '-' + \ optimizer_name + '-' + lr_scheduler_name + '-' + split_method + '-' + str(seed)) os.makedirs(checkpoint_folder, exist_ok=True) checkpoint_filename = 'fold_' + str(fold) + "_checkpoint.pth" checkpoint_filepath = os.path.join(checkpoint_folder, checkpoint_filename) log = Logger() log.open(os.path.join(checkpoint_folder, 'fold_' + str(fold) + '_log_train.txt'), mode='a+') log.write('\t%s\n' % COMMON_STRING) log.write('\n') log.write('\tseed = %u\n' % seed) log.write('\tFOLD = %s\n' % fold) log.write('\t__file__ = %s\n' % __file__) log.write('\tout_dir = %s\n' % checkpoint_folder) log.write('weight_grapheme_root\n %s\n' % (str(weight_grapheme_root))) log.write('weight_vowel_diacritic\n %s\n' % (str(weight_vowel_diacritic))) log.write('weight_consonant_diacritic\n %s\n' % (str(weight_consonant_diacritic))) log.write('weight_grapheme\n %s\n' % (str(weight_grapheme))) log.write('\n') ############################################################################## define unet model with backbone def load(model, pretrain_file, skip=[]): pretrain_state_dict = torch.load(pretrain_file) state_dict = model.state_dict() keys = list(state_dict.keys()) for key in keys: if any(s in key for s in skip): continue try: state_dict[key] = pretrain_state_dict[key] except: print(key) model.load_state_dict(state_dict) return model ############################################################################### model model = BengaliaiNet(model_type=model_type, \ n_classes=NUM_CLASSES) model = model.cuda() if load_pretrain: print("Load pretrain model") model = load(model, checkpoint_filepath, skip=[ 'avg_poolings.0.p', 'avg_poolings.1.p', 'avg_poolings.2.p', 'avg_poolings.3.p' ]) # model = load(model, checkpoint_filepath, skip=[]) ############################################################################### optimizer # param_optimizer = list(model.parameters()) # param_optimizer = list(model.named_parameters()) if freeze_first: param_optimizer = list(model.head.named_parameters()) + list( model.tail.named_parameters()) + list( model.logits.named_parameters()) else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \ 'lr': lr, \ 'weight_decay': WEIGHT_DECAY}, \ {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \ 'lr': lr, \ 'weight_decay': 0.0} ] if optimizer_name == "Adam": optimizer = torch.optim.Adam(optimizer_grouped_parameters) min_lr = 1e-4 elif optimizer_name == "SGD": optimizer = torch.optim.SGD(optimizer_grouped_parameters, momentum=0.5) min_lr = 1e-2 elif optimizer_name == "Ranger": optimizer = Ranger(optimizer_grouped_parameters) min_lr = 1e-4 elif optimizer_name == "RAdam": optimizer = RAdam(optimizer_grouped_parameters) min_lr = 1e-4 elif optimizer_name == "AdamW": optimizer = AdamW(optimizer_grouped_parameters, eps=4e-5) min_lr = 1e-4 elif optimizer_name == "FusedAdam": optimizer = FusedAdam(optimizer_grouped_parameters, bias_correction=False) min_lr = 1e-4 else: raise NotImplementedError # add swa # optimizer = torchcontrib.optim.SWA(optimizer, swa_start=int(len(train_data_loader) / 10), \ # swa_freq=int(len(train_data_loader) / 10), swa_lr=0.05) # optimizer = torchcontrib.optim.SWA(optimizer) ############################################################################### lr_scheduler if lr_scheduler_name == "WarmupCosineAnealing": num_train_optimization_steps = num_epoch * len( train_data_loader) // accumulation_steps scheduler = get_cosine_schedule_with_warmup(optimizer, \ num_warmup_steps=int(num_train_optimization_steps*warmup_proportion), \ num_training_steps=num_train_optimization_steps) lr_scheduler_each_iter = True elif lr_scheduler_name == "WarmupLinearSchedule": num_train_optimization_steps = num_epoch * len( train_data_loader) // accumulation_steps scheduler = get_linear_schedule_with_warmup(optimizer, \ num_warmup_steps=int(num_train_optimization_steps*warmup_proportion), \ num_training_steps=num_train_optimization_steps) lr_scheduler_each_iter = True elif lr_scheduler_name == "WarmupCosineAnealingWithHardRestart": num_train_optimization_steps = num_epoch * len( train_data_loader) // accumulation_steps scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, \ num_warmup_steps=int(num_train_optimization_steps*warmup_proportion), \ num_training_steps=num_train_optimization_steps) lr_scheduler_each_iter = True elif lr_scheduler_name == "ReduceLROnPlateau": scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.6, patience=3, min_lr=min_lr) lr_scheduler_each_iter = False elif lr_scheduler_name == "OneCycleLR": scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr, total_steps=None, epochs=num_epoch, \ steps_per_epoch=int(len(train_data_loader)/accumulation_steps), \ pct_start=0.3, anneal_strategy='cos', cycle_momentum=True, \ base_momentum=0.85, max_momentum=0.95, div_factor=25.0, final_div_factor=10000.0, last_epoch=-1) lr_scheduler_each_iter = True elif lr_scheduler_name == "CycleLR": step_size = len(train_data_loader) * 5 / (accumulation_steps) base_lr, max_lr = lr / 6, lr scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr, step_size_up=step_size, mode='exp_range', gamma=0.99994) lr_scheduler_each_iter = True else: raise NotImplementedError log.write('net\n %s\n' % (model_type)) log.write('optimizer\n %s\n' % (optimizer_name)) log.write('schduler\n %s\n' % (lr_scheduler_name)) log.write('\n') ############################################################################### mix precision if apex: model, optimizer = amp.initialize(model, optimizer, opt_level="O1") # model = nn.DataParallel(model) ############################################################################### eval setting eval_step = len(train_data_loader) # or len(train_data_loader) log_step = int(len(train_data_loader) / 20) eval_count = 0 count = 0 ############################################################################### training log.write('** start training here! **\n') log.write(' batch_size=%d, accumulation_steps=%d\n' % (batch_size, accumulation_steps)) log.write(' experiment = %s\n' % str(__file__.split('/')[-2:])) valid_loss = np.zeros(1, np.float32) train_loss = np.zeros(1, np.float32) valid_metric_optimal = -np.inf # define tensorboard writer and timer writer = SummaryWriter() # define criterion if loss_type == 'mse': criterion = nn.MSELoss() elif loss_type == 'mae': criterion = nn.SmoothL1Loss() elif loss_type == 'bce': criterion = nn.BCEWithLogitsLoss() elif loss_type == 'focal': criterion = FocalLoss() elif loss_type == 'focalonehot': criterion = FocalOnehotLoss() elif loss_type == 'ceonehot': criterion = CrossEntropyOnehotLoss() elif loss_type == "ceonehotohem": criterion = CrossEntropyOnehotLossOHEM(top_k=0.8) elif loss_type == "focalonehotohem": criterion = FocalOnehotLossOHEM(top_k=0.8) else: raise NotImplementedError for epoch in range(1, num_epoch + 1): # init cache torch.cuda.empty_cache() if freeze_first: # if epoch < freeze_epoches: # for param_group in optimizer.param_groups: # param_group['lr'] = 1e-4 if epoch == freeze_epoches: optimizer.add_param_group( {'params': model.basemodel.parameters()}) for param_group in optimizer.param_groups: param_group['lr'] = lr # if (epoch + 1) % 10 == 0: # for param_group in optimizer.param_groups: # param_group['lr'] = lr / 1.5 # init in-epoch statistics grapheme_root_train = [] vowel_diacritic_train = [] consonant_diacritic_train = [] grapheme_train = [] grapheme_root_prediction_train = [] vowel_diacritic_prediction_train = [] consonant_diacritic_prediction_train = [] grapheme_prediction_train = [] # update lr and start from start_epoch if lr_scheduler_each_iter: if (epoch < start_epoch): for _ in range(len(train_data_loader)): scheduler.step() continue else: if lr_scheduler_name != "ReduceLROnPlateau": scheduler.step() if (epoch < start_epoch): continue log.write("Epoch%s\n" % epoch) log.write('\n') sum_train_loss = np.zeros_like(train_loss) sum_train = np.zeros_like(train_loss) # init optimizer torch.cuda.empty_cache() model.zero_grad() for tr_batch_i, (image, grapheme_root, vowel_diacritic, consonant_diacritic, grapheme) in enumerate(train_data_loader): rate = 0 for param_group in optimizer.param_groups: rate += param_group['lr'] / len(optimizer.param_groups) # set model training mode model.train() # set input to cuda mode image = image.cuda() image_copy = image.clone() grapheme_root = grapheme_root.cuda().float() vowel_diacritic = vowel_diacritic.cuda().float() consonant_diacritic = consonant_diacritic.cuda().float() grapheme = grapheme.cuda().float() if ((loss_type == 'mae') or (loss_type == 'mse')): _, grapheme_root = torch.max(grapheme_root, 1) grapheme_root = torch.squeeze(grapheme_root.float()) _, vowel_diacritic = torch.max(vowel_diacritic, 1) vowel_diacritic = torch.squeeze(vowel_diacritic.float()) _, consonant_diacritic = torch.max(consonant_diacritic, 1) consonant_diacritic = torch.squeeze( consonant_diacritic.float()) _, grapheme = torch.max(grapheme, 1) grapheme = torch.squeeze(grapheme.float()) # predict and calculate loss (cutmix added) r = np.random.rand(1) if r < cutmix_prob: def rand_bbox(size, lam): W = size[2] H = size[3] cut_rat = np.sqrt(1. - lam) cut_w = np.int(W * cut_rat) cut_h = np.int(H * cut_rat) # uniform cx = np.random.randint(W) cy = np.random.randint(H) bbx1 = np.clip(cx - cut_w // 2, 0, W) bby1 = np.clip(cy - cut_h // 2, 0, H) bbx2 = np.clip(cx + cut_w // 2, 0, W) bby2 = np.clip(cy + cut_h // 2, 0, H) return bbx1, bby1, bbx2, bby2 # generate mixed sample lam = np.random.beta(beta, beta) rand_index = torch.randperm(image.shape[0]).cuda() # shuffled_image = image[rand_index] grapheme_root_a = grapheme_root grapheme_root_b = grapheme_root[rand_index] vowel_diacritic_a = vowel_diacritic vowel_diacritic_b = vowel_diacritic[rand_index] consonant_diacritic_a = consonant_diacritic consonant_diacritic_b = consonant_diacritic[rand_index] grapheme_a = grapheme grapheme_b = grapheme[rand_index] bbx1, bby1, bbx2, bby2 = rand_bbox(image.shape, lam) image[:, :, bbx1:bbx2, bby1:bby2] = image[rand_index, :, bbx1:bbx2, bby1:bby2] # adjust lambda to exactly match pixel ratio lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (image.size()[-1] * image.size()[-2])) # lam = 1 - (torch.mean(image[:, :, bbx1:bbx2, bby1:bby2]) / torch.mean(image)) # compute output predictions = model(image) grapheme_root_prediction = torch.squeeze(predictions[0]) vowel_diacritic_prediction = torch.squeeze(predictions[1]) consonant_diacritic_prediction = torch.squeeze(predictions[2]) grapheme_prediction = torch.squeeze(predictions[3]) loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root_a) * lam + \ weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root_b) * (1 - lam) + \ weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic_a) * lam + \ weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic_b) * (1- lam) + \ weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic_a) * lam + \ weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic_b) * (1 - lam) + \ weight_grapheme * criterion(grapheme_prediction, grapheme_a) * lam + \ weight_grapheme * criterion(grapheme_prediction, grapheme_b) * (1 - lam) elif r < (mixup_prob + cutmix_prob): indices = torch.randperm(image.size(0)) shuffled_image = image[indices] shuffled_grapheme_root = grapheme_root[indices] shuffled_vowel_diacritic = vowel_diacritic[indices] shuffled_consonant_diacritic = consonant_diacritic[indices] shuffled_grapheme = grapheme[indices] lam = np.random.beta(alpha, alpha) image = image * lam + shuffled_image * (1 - lam) predictions = model(image) grapheme_root_prediction = torch.squeeze(predictions[0]) vowel_diacritic_prediction = torch.squeeze(predictions[1]) consonant_diacritic_prediction = torch.squeeze(predictions[2]) grapheme_prediction = torch.squeeze(predictions[3]) loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root) * lam + \ weight_grapheme_root * criterion(grapheme_root_prediction, shuffled_grapheme_root) * (1 - lam) + \ weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic) * lam + \ weight_vowel_diacritic * criterion(vowel_diacritic_prediction, shuffled_vowel_diacritic) * (1- lam) + \ weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic) * lam + \ weight_consonant_diacritic * criterion(consonant_diacritic_prediction, shuffled_consonant_diacritic) * (1 - lam) + \ weight_grapheme * criterion(grapheme_prediction, grapheme) * lam + \ weight_grapheme * criterion(grapheme_prediction, shuffled_grapheme) * (1 - lam) else: predictions = model(image) grapheme_root_prediction = torch.squeeze(predictions[0]) vowel_diacritic_prediction = torch.squeeze(predictions[1]) consonant_diacritic_prediction = torch.squeeze(predictions[2]) grapheme_prediction = torch.squeeze(predictions[3]) loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root) + \ weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic) + \ weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic) + \ weight_grapheme * criterion(grapheme_prediction, grapheme) # use apex if apex: with amp.scale_loss(loss / accumulation_steps, optimizer) as scaled_loss: scaled_loss.backward() else: # don't use apex loss.backward() if ((tr_batch_i + 1) % accumulation_steps == 0): torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2) optimizer.step() # if (tr_batch_i+1) > int(len(train_data_loader) / 10) and (tr_batch_i+1) % int(len(train_data_loader) / 5) == 0: # optimizer.update_swa() model.zero_grad() # adjust lr if (lr_scheduler_each_iter): scheduler.step() writer.add_scalar( 'train_loss_' + str(fold), loss.item(), (epoch - 1) * len(train_data_loader) * batch_size + tr_batch_i * batch_size) # calculate statistics if not ((loss_type == "mae") or (loss_type == "mse")): grapheme_root = torch.argmax(grapheme_root, 1) grapheme_root = torch.squeeze(grapheme_root.float()) vowel_diacritic = torch.argmax(vowel_diacritic, 1) vowel_diacritic = torch.squeeze(vowel_diacritic.float()) consonant_diacritic = torch.argmax(consonant_diacritic, 1) consonant_diacritic = torch.squeeze( consonant_diacritic.float()) grapheme = torch.argmax(grapheme, 1) grapheme = torch.squeeze(grapheme.float()) # calculate traing result without cutmix, otherwise prediction is not related to image with torch.no_grad(): predictions_no_cutmix = model(image_copy) grapheme_root_prediction_no_cutmix = torch.squeeze( predictions_no_cutmix[0]) vowel_diacritic_prediction_no_cutmix = torch.squeeze( predictions_no_cutmix[1]) consonant_diacritic_prediction_no_cutmix = torch.squeeze( predictions_no_cutmix[2]) grapheme_prediction_no_cutmix = torch.squeeze( predictions_no_cutmix[3]) # calculate statistics if not ((loss_type == "mae") or (loss_type == "mse")): grapheme_root_prediction_no_cutmix = F.softmax( grapheme_root_prediction_no_cutmix, 1) grapheme_root_prediction_no_cutmix = torch.argmax( grapheme_root_prediction_no_cutmix, 1) grapheme_root_prediction_no_cutmix = torch.squeeze( grapheme_root_prediction_no_cutmix.float()) vowel_diacritic_prediction_no_cutmix = F.softmax( vowel_diacritic_prediction_no_cutmix, 1) vowel_diacritic_prediction_no_cutmix = torch.argmax( vowel_diacritic_prediction_no_cutmix, 1) vowel_diacritic_prediction_no_cutmix = torch.squeeze( vowel_diacritic_prediction_no_cutmix.float()) consonant_diacritic_prediction_no_cutmix = F.softmax( consonant_diacritic_prediction_no_cutmix, 1) consonant_diacritic_prediction_no_cutmix = torch.argmax( consonant_diacritic_prediction_no_cutmix, 1) consonant_diacritic_prediction_no_cutmix = torch.squeeze( consonant_diacritic_prediction_no_cutmix.float()) grapheme_prediction_no_cutmix = F.softmax( grapheme_prediction_no_cutmix, 1) grapheme_prediction_no_cutmix = torch.argmax( grapheme_prediction_no_cutmix, 1) grapheme_prediction_no_cutmix = torch.squeeze( grapheme_prediction_no_cutmix.float()) grapheme_root = grapheme_root.cpu().detach().numpy() vowel_diacritic = vowel_diacritic.cpu().detach().numpy() consonant_diacritic = consonant_diacritic.cpu().detach().numpy() grapheme = grapheme.cpu().detach().numpy() grapheme_root_prediction = grapheme_root_prediction_no_cutmix.cpu( ).detach().numpy() vowel_diacritic_prediction = vowel_diacritic_prediction_no_cutmix.cpu( ).detach().numpy() consonant_diacritic_prediction = consonant_diacritic_prediction_no_cutmix.cpu( ).detach().numpy() grapheme_prediction = grapheme_prediction_no_cutmix.cpu().detach( ).numpy() l = np.array([loss.item() * batch_size]) n = np.array([batch_size]) sum_train_loss = sum_train_loss + l sum_train = sum_train + n grapheme_root_train.append(grapheme_root) vowel_diacritic_train.append(vowel_diacritic) consonant_diacritic_train.append(consonant_diacritic) grapheme_train.append(grapheme) grapheme_root_prediction_train.append(grapheme_root_prediction) vowel_diacritic_prediction_train.append(vowel_diacritic_prediction) consonant_diacritic_prediction_train.append( consonant_diacritic_prediction) grapheme_prediction_train.append(grapheme_prediction) grapheme_root_recall_train = metric(np.concatenate(grapheme_root_prediction_train, axis=0), \ np.concatenate(grapheme_root_train, axis=0)) vowel_diacritic_recall_train = metric(np.concatenate(vowel_diacritic_prediction_train, axis=0), \ np.concatenate(vowel_diacritic_train, axis=0)) consonant_diacritic_recall_train = metric(np.concatenate(consonant_diacritic_prediction_train, axis=0), \ np.concatenate(consonant_diacritic_train, axis=0)) grapheme_recall_train = metric(np.concatenate(grapheme_prediction_train, axis=0), \ np.concatenate(grapheme_train, axis=0)) average_recall_train = np.average([grapheme_root_recall_train, vowel_diacritic_recall_train, consonant_diacritic_recall_train], \ weights=[2,1,1]) # log for training if (tr_batch_i + 1) % log_step == 0: train_loss = sum_train_loss / (sum_train + 1e-12) sum_train_loss[...] = 0 sum_train[...] = 0 log.write('lr: %f train loss: %f average_recall: %f grapheme_root_recall: %f vowel_diacritic_recall: %f consonant_diacritic_recall: %f grapheme_recall: %f\n' % \ (rate, train_loss[0], average_recall_train, \ grapheme_root_recall_train, \ vowel_diacritic_recall_train, \ consonant_diacritic_recall_train, \ grapheme_recall_train)) if (tr_batch_i + 1) % eval_step == 0: # finish training then update swa # optimizer.bn_update(train_data_loader, model) # optimizer.swap_swa_sgd() eval_count += 1 valid_loss = np.zeros(1, np.float32) valid_num = np.zeros_like(valid_loss) grapheme_root_val = [] vowel_diacritic_val = [] consonant_diacritic_val = [] grapheme_val = [] grapheme_root_prediction_val = [] vowel_diacritic_prediction_val = [] consonant_diacritic_prediction_val = [] grapheme_prediction_val = [] grapheme_root_prediction_decode_val = [] vowel_diacritic_prediction_decode_val = [] consonant_diacritic_prediction_decode_val = [] grapheme_prediction_decode_val = [] with torch.no_grad(): # init cache torch.cuda.empty_cache() for val_batch_i, (image, grapheme_root, vowel_diacritic, consonant_diacritic, grapheme) in enumerate(val_data_loader): # set model to eval mode model.eval() # set input to cuda mode image = image.cuda() grapheme_root = grapheme_root.cuda().float() vowel_diacritic = vowel_diacritic.cuda().float() consonant_diacritic = consonant_diacritic.cuda().float( ) grapheme = grapheme.cuda().float() if ((loss_type == 'mae') or (loss_type == 'mse')): _, grapheme_root = torch.max(grapheme_root, 1) grapheme_root = torch.squeeze( grapheme_root.float()) _, vowel_diacritic = torch.max(vowel_diacritic, 1) vowel_diacritic = torch.squeeze( vowel_diacritic.float()) _, consonant_diacritic = torch.max( consonant_diacritic, 1) consonant_diacritic = torch.squeeze( consonant_diacritic.float()) _, grapheme = torch.max(grapheme, 1) grapheme = torch.squeeze(grapheme.float()) # predict and calculate loss (only need torch.sigmoid when inference) predictions = model(image) grapheme_root_prediction = torch.squeeze( predictions[0]) vowel_diacritic_prediction = torch.squeeze( predictions[1]) consonant_diacritic_prediction = torch.squeeze( predictions[2]) grapheme_prediction = torch.squeeze(predictions[3]) loss = weight_grapheme_root * criterion(grapheme_root_prediction, grapheme_root) + \ weight_vowel_diacritic * criterion(vowel_diacritic_prediction, vowel_diacritic) + \ weight_consonant_diacritic * criterion(consonant_diacritic_prediction, consonant_diacritic) + \ weight_grapheme * criterion(grapheme_prediction, grapheme) writer.add_scalar( 'val_loss_' + str(fold), loss.item(), (eval_count - 1) * len(val_data_loader) * valid_batch_size + val_batch_i * valid_batch_size) # calculate statistics if not ((loss_type == "mae") or (loss_type == "mse")): grapheme_root = torch.argmax(grapheme_root, 1) grapheme_root = torch.squeeze( grapheme_root.float()) vowel_diacritic = torch.argmax(vowel_diacritic, 1) vowel_diacritic = torch.squeeze( vowel_diacritic.float()) consonant_diacritic = torch.argmax( consonant_diacritic, 1) consonant_diacritic = torch.squeeze( consonant_diacritic.float()) grapheme = torch.argmax(grapheme, 1) grapheme = torch.squeeze(grapheme.float()) grapheme_root_prediction = F.softmax( grapheme_root_prediction, 1) grapheme_root_prediction = torch.argmax( grapheme_root_prediction, 1) grapheme_root_prediction = torch.squeeze( grapheme_root_prediction.float()) vowel_diacritic_prediction = F.softmax( vowel_diacritic_prediction, 1) vowel_diacritic_prediction = torch.argmax( vowel_diacritic_prediction, 1) vowel_diacritic_prediction = torch.squeeze( vowel_diacritic_prediction.float()) consonant_diacritic_prediction = F.softmax( consonant_diacritic_prediction, 1) consonant_diacritic_prediction = torch.argmax( consonant_diacritic_prediction, 1) consonant_diacritic_prediction = torch.squeeze( consonant_diacritic_prediction.float()) grapheme_prediction = F.softmax( grapheme_prediction, 1) grapheme_prediction = torch.argmax( grapheme_prediction, 1) grapheme_prediction = torch.squeeze( grapheme_prediction.float()) grapheme_root = grapheme_root.cpu().detach().numpy() vowel_diacritic = vowel_diacritic.cpu().detach().numpy( ) consonant_diacritic = consonant_diacritic.cpu().detach( ).numpy() grapheme = grapheme.cpu().detach().numpy() grapheme_root_prediction = grapheme_root_prediction.cpu( ).detach().numpy() vowel_diacritic_prediction = vowel_diacritic_prediction.cpu( ).detach().numpy() consonant_diacritic_prediction = consonant_diacritic_prediction.cpu( ).detach().numpy() grapheme_prediction = grapheme_prediction.cpu().detach( ).numpy() grapheme_root_prediction_decode, \ vowel_diacritic_prediction_decode, \ consonant_diacritic_prediction_decode, \ grapheme_prediction_decode = decode([grapheme_root_prediction.copy(), vowel_diacritic_prediction.copy(), \ consonant_diacritic_prediction.copy(), grapheme_prediction.copy()], CLASS_MAP) l = np.array([loss.item() * valid_batch_size]) n = np.array([valid_batch_size]) valid_loss = valid_loss + l valid_num = valid_num + n grapheme_root_val.append(grapheme_root) vowel_diacritic_val.append(vowel_diacritic) consonant_diacritic_val.append(consonant_diacritic) grapheme_val.append(grapheme) grapheme_root_prediction_val.append( grapheme_root_prediction) vowel_diacritic_prediction_val.append( vowel_diacritic_prediction) consonant_diacritic_prediction_val.append( consonant_diacritic_prediction) grapheme_prediction_val.append(grapheme_prediction) grapheme_root_prediction_decode_val.append( grapheme_root_prediction_decode) vowel_diacritic_prediction_decode_val.append( vowel_diacritic_prediction_decode) consonant_diacritic_prediction_decode_val.append( consonant_diacritic_prediction_decode) grapheme_prediction_decode_val.append( grapheme_prediction_decode) grapheme_root_recall_val = metric(np.concatenate(grapheme_root_prediction_val, axis=0), \ np.concatenate(grapheme_root_val, axis=0)) vowel_diacritic_recall_val = metric(np.concatenate(vowel_diacritic_prediction_val, axis=0), \ np.concatenate(vowel_diacritic_val, axis=0)) consonant_diacritic_recall_val = metric(np.concatenate(consonant_diacritic_prediction_val, axis=0), \ np.concatenate(consonant_diacritic_val, axis=0)) grapheme_recall_val = metric(np.concatenate(grapheme_prediction_val, axis=0), \ np.concatenate(grapheme_val, axis=0)) grapheme_root_recall_decode_val = metric(np.concatenate(grapheme_root_prediction_decode_val, axis=0), \ np.concatenate(grapheme_root_val, axis=0)) vowel_diacritic_recall_decode_val = metric(np.concatenate(vowel_diacritic_prediction_decode_val, axis=0), \ np.concatenate(vowel_diacritic_val, axis=0)) consonant_diacritic_recall_decode_val = metric(np.concatenate(consonant_diacritic_prediction_decode_val, axis=0), \ np.concatenate(consonant_diacritic_val, axis=0)) average_recall_val = np.average([grapheme_root_recall_val, vowel_diacritic_recall_val, consonant_diacritic_recall_val], \ weights=[2,1,1]) average_recall_decode_val = np.average([grapheme_root_recall_decode_val, vowel_diacritic_recall_decode_val, \ consonant_diacritic_recall_decode_val], \ weights=[2,1,1]) valid_loss = valid_loss / valid_num log.write('valid loss: %f average_recall: %f grapheme_root_recall: %f vowel_diacritic_recall: %f consonant_diacritic_recall: %f grapheme_recall: %f average_recall_decode: %f grapheme_root_recall_decode: %f vowel_diacritic_recall_decode: %f consonant_diacritic_recall_decode: %f\n' % \ (valid_loss[0], average_recall_val, \ grapheme_root_recall_val, \ vowel_diacritic_recall_val, \ consonant_diacritic_recall_val, \ grapheme_recall_val, \ average_recall_decode_val, \ grapheme_root_recall_decode_val, \ vowel_diacritic_recall_decode_val, \ consonant_diacritic_recall_decode_val)) val_metric_epoch = average_recall_val # scheduler lr by metric if lr_scheduler_name == "ReduceLROnPlateau": scheduler.step(val_metric_epoch) if (val_metric_epoch >= valid_metric_optimal): log.write('Validation metric improved ({:.6f} --> {:.6f}). Saving model ...'.format(\ valid_metric_optimal, val_metric_epoch)) valid_metric_optimal = val_metric_epoch torch.save(model.state_dict(), checkpoint_filepath) count = 0 else: count += 1 if (count == early_stopping): break
def tune_mahalanobis_hyperparams(): def print_tuning_results(results, stypes): mtypes = ['FPR', 'DTERR', 'AUROC', 'AUIN', 'AUOUT'] for stype in stypes: print(' OOD detection method: ' + stype) for mtype in mtypes: print(' {mtype:6s}'.format(mtype=mtype), end='') print('\n{val:6.2f}'.format(val=100. * results[stype]['FPR']), end='') print(' {val:6.2f}'.format(val=100. * results[stype]['DTERR']), end='') print(' {val:6.2f}'.format(val=100. * results[stype]['AUROC']), end='') print(' {val:6.2f}'.format(val=100. * results[stype]['AUIN']), end='') print(' {val:6.2f}\n'.format(val=100. * results[stype]['AUOUT']), end='') print('') print('Tuning hyper-parameters...') stypes = ['mahalanobis'] save_dir = os.path.join('output/hyperparams/', args.name, 'tmp') if not os.path.exists(save_dir): os.makedirs(save_dir) normalizer = transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)) transform = transforms.Compose([ transforms.ToTensor(), ]) if args.in_dataset == "CIFAR-10": trainset = torchvision.datasets.CIFAR10('./datasets/cifar10', train=True, download=True, transform=transform) trainloaderIn = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=2) testset = torchvision.datasets.CIFAR10(root='./datasets/cifar10', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=True, num_workers=2) num_classes = 10 elif args.in_dataset == "CIFAR-100": trainset = torchvision.datasets.CIFAR100('./datasets/cifar10', train=True, download=True, transform=transform) trainloaderIn = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=2) testset = torchvision.datasets.CIFAR100(root='./datasets/cifar100', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=True, num_workers=2) num_classes = 100 valloaderOut = torch.utils.data.DataLoader( TinyImages(transform=transforms.Compose([ transforms.ToTensor(), transforms.ToPILImage(), transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor() ])), batch_size=args.batch_size, shuffle=True, num_workers=2) model = dn.DenseNet3(args.layers, num_classes, normalizer=normalizer) checkpoint = torch.load( "./checkpoints/{name}/checkpoint_{epochs}.pth.tar".format( name=args.name, epochs=args.epochs)) model.load_state_dict(checkpoint['state_dict']) model.eval() model.cuda() # set information about feature extaction temp_x = torch.rand(2, 3, 32, 32) temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = sample_estimator(model, num_classes, feature_list, trainloaderIn) print('train logistic regression model') m = 1000 val_in = [] val_out = [] cnt = 0 for data, target in trainloaderIn: for x in data: val_in.append(x.numpy()) cnt += 1 if cnt == m: break if cnt == m: break cnt = 0 for data, target in valloaderOut: for x in data: val_out.append(data[0].numpy()) cnt += 1 if cnt == m: break if cnt == m: break train_lr_data = [] train_lr_label = [] train_lr_data.extend(val_in) train_lr_label.extend(np.zeros(m)) train_lr_data.extend(val_out) train_lr_label.extend(np.ones(m)) train_lr_data = torch.tensor(train_lr_data) train_lr_label = torch.tensor(train_lr_label) best_fpr = 1.1 best_magnitude = 0.0 for magnitude in np.arange(0, 0.0041, 0.004 / 20): train_lr_Mahalanobis = [] total = 0 for data_index in range( int(np.floor(train_lr_data.size(0) / args.batch_size))): data = train_lr_data[total:total + args.batch_size] total += args.batch_size Mahalanobis_scores = get_Mahalanobis_score(model, data, num_classes, sample_mean, precision, num_output, magnitude) train_lr_Mahalanobis.extend(Mahalanobis_scores) train_lr_Mahalanobis = np.asarray(train_lr_Mahalanobis, dtype=np.float32) regressor = LogisticRegressionCV().fit(train_lr_Mahalanobis, train_lr_label) print('Logistic Regressor params:', regressor.coef_, regressor.intercept_) t0 = time.time() f1 = open(os.path.join(save_dir, "confidence_mahalanobis_In.txt"), 'w') f2 = open(os.path.join(save_dir, "confidence_mahalanobis_Out.txt"), 'w') ########################################In-distribution########################################### print("Processing in-distribution images") count = 0 for i in range(int(m / args.batch_size) + 1): if i * args.batch_size >= m: break images = torch.tensor( val_in[i * args.batch_size:min((i + 1) * args.batch_size, m)]) # if j<1000: continue batch_size = images.shape[0] Mahalanobis_scores = get_Mahalanobis_score(model, images, num_classes, sample_mean, precision, num_output, magnitude) confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:, 1] for k in range(batch_size): f1.write("{}\n".format(-confidence_scores[k])) count += batch_size print("{:4}/{:4} images processed, {:.1f} seconds used.".format( count, m, time.time() - t0)) t0 = time.time() ###################################Out-of-Distributions##################################### t0 = time.time() print("Processing out-of-distribution images") count = 0 for i in range(int(m / args.batch_size) + 1): if i * args.batch_size >= m: break images = torch.tensor( val_out[i * args.batch_size:min((i + 1) * args.batch_size, m)]) # if j<1000: continue batch_size = images.shape[0] Mahalanobis_scores = get_Mahalanobis_score(model, images, num_classes, sample_mean, precision, num_output, magnitude) confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:, 1] for k in range(batch_size): f2.write("{}\n".format(-confidence_scores[k])) count += batch_size print("{:4}/{:4} images processed, {:.1f} seconds used.".format( count, m, time.time() - t0)) t0 = time.time() f1.close() f2.close() results = metric(save_dir, stypes) print_tuning_results(results, stypes) fpr = results['mahalanobis']['FPR'] if fpr < best_fpr: best_fpr = fpr best_magnitude = magnitude best_regressor = regressor print('Best Logistic Regressor params:', best_regressor.coef_, best_regressor.intercept_) print('Best magnitude', best_magnitude) return sample_mean, precision, best_regressor, best_magnitude
def eval_msp_and_odin(): stypes = ['MSP', 'ODIN'] save_dir = os.path.join('output/ood_scores/', args.out_dataset, args.name, 'adv' if args.adv else 'nat') if not os.path.exists(save_dir): os.makedirs(save_dir) start = time.time() #loading data sets normalizer = transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)) transform = transforms.Compose([ transforms.ToTensor(), ]) if args.in_dataset == "CIFAR-10": testset = torchvision.datasets.CIFAR10(root='./datasets/cifar10', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=True, num_workers=2) num_classes = 10 elif args.in_dataset == "CIFAR-100": testset = torchvision.datasets.CIFAR100(root='./datasets/cifar100', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=True, num_workers=2) num_classes = 100 model = dn.DenseNet3(args.layers, num_classes, normalizer=normalizer) checkpoint = torch.load( "./checkpoints/{name}/checkpoint_{epochs}.pth.tar".format( name=args.name, epochs=args.epochs)) model.load_state_dict(checkpoint['state_dict']) model.eval() model.cuda() if args.out_dataset == 'SVHN': testsetout = svhn.SVHN('datasets/ood_datasets/svhn/', split='test', transform=transforms.ToTensor(), download=False) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) elif args.out_dataset == 'dtd': testsetout = torchvision.datasets.ImageFolder( root="datasets/ood_datasets/dtd/images", transform=transforms.Compose([ transforms.Resize(32), transforms.CenterCrop(32), transforms.ToTensor() ])) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) elif args.out_dataset == 'places365': testsetout = torchvision.datasets.ImageFolder( root="datasets/ood_datasets/places365/test_subset", transform=transforms.Compose([ transforms.Resize(32), transforms.CenterCrop(32), transforms.ToTensor() ])) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) else: testsetout = torchvision.datasets.ImageFolder( "./datasets/ood_datasets/{}".format(args.out_dataset), transform=transform) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) t0 = time.time() f1 = open(os.path.join(save_dir, "confidence_MSP_In.txt"), 'w') f2 = open(os.path.join(save_dir, "confidence_MSP_Out.txt"), 'w') g1 = open(os.path.join(save_dir, "confidence_ODIN_In.txt"), 'w') g2 = open(os.path.join(save_dir, "confidence_ODIN_Out.txt"), 'w') N = 10000 if args.out_dataset == "iSUN": N = 8925 if args.out_dataset == "dtd": N = 5640 ########################################In-distribution########################################### print("Processing in-distribution images") if args.adv: attack = ConfidenceLinfPGDAttack(model, eps=args.epsilon, nb_iter=args.iters, eps_iter=args.iter_size, rand_init=True, clip_min=0., clip_max=1., in_distribution=True, num_classes=num_classes) count = 0 for j, data in enumerate(testloaderIn): images, _ = data batch_size = images.shape[0] if count + batch_size > N: images = images[:N - count] batch_size = images.shape[0] if args.adv: adv_images = attack.perturb(images) inputs = Variable(adv_images, requires_grad=True) else: inputs = Variable(images, requires_grad=True) outputs = model(inputs) nnOutputs = MSP(outputs, model) for k in range(batch_size): f1.write("{}\n".format(np.max(nnOutputs[k]))) nnOutputs = ODIN(inputs, outputs, model, temper=args.temperature, noiseMagnitude1=args.magnitude) for k in range(batch_size): g1.write("{}\n".format(np.max(nnOutputs[k]))) count += batch_size print("{:4}/{:4} images processed, {:.1f} seconds used.".format( count, N, time.time() - t0)) t0 = time.time() if count == N: break ###################################Out-of-Distributions##################################### t0 = time.time() print("Processing out-of-distribution images") if args.adv: attack = ConfidenceLinfPGDAttack(model, eps=args.epsilon, nb_iter=args.iters, eps_iter=args.iter_size, rand_init=True, clip_min=0., clip_max=1., in_distribution=False, num_classes=num_classes) count = 0 for j, data in enumerate(testloaderOut): images, labels = data batch_size = images.shape[0] if args.adv: adv_images = attack.perturb(images) inputs = Variable(adv_images, requires_grad=True) else: inputs = Variable(images, requires_grad=True) outputs = model(inputs) nnOutputs = MSP(outputs, model) for k in range(batch_size): f2.write("{}\n".format(np.max(nnOutputs[k]))) nnOutputs = ODIN(inputs, outputs, model, temper=args.temperature, noiseMagnitude1=args.magnitude) for k in range(batch_size): g2.write("{}\n".format(np.max(nnOutputs[k]))) count += batch_size print("{:4}/{:4} images processed, {:.1f} seconds used.".format( count, N, time.time() - t0)) t0 = time.time() if count == N: break f1.close() f2.close() g1.close() g2.close() results = metric(save_dir, stypes) print_results(results, stypes)
def eval_mahalanobis(sample_mean, precision, regressor, magnitude): stypes = ['mahalanobis'] save_dir = os.path.join('output/ood_scores/', args.out_dataset, args.name, 'adv' if args.adv else 'nat') if not os.path.exists(save_dir): os.makedirs(save_dir) start = time.time() #loading data sets normalizer = transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)) transform = transforms.Compose([ transforms.ToTensor(), ]) if args.in_dataset == "CIFAR-10": trainset = torchvision.datasets.CIFAR10('./datasets/cifar10', train=True, download=True, transform=transform) trainloaderIn = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=2) testset = torchvision.datasets.CIFAR10(root='./datasets/cifar10', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=True, num_workers=2) num_classes = 10 elif args.in_dataset == "CIFAR-100": trainset = torchvision.datasets.CIFAR100('./datasets/cifar10', train=True, download=True, transform=transform) trainloaderIn = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=2) testset = torchvision.datasets.CIFAR100(root='./datasets/cifar100', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=True, num_workers=2) num_classes = 100 model = dn.DenseNet3(args.layers, num_classes, normalizer=normalizer) checkpoint = torch.load( "./checkpoints/{name}/checkpoint_{epochs}.pth.tar".format( name=args.name, epochs=args.epochs)) model.load_state_dict(checkpoint['state_dict']) model.eval() model.cuda() if args.out_dataset == 'SVHN': testsetout = svhn.SVHN('datasets/ood_datasets/svhn/', split='test', transform=transforms.ToTensor(), download=False) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) elif args.out_dataset == 'dtd': testsetout = torchvision.datasets.ImageFolder( root="datasets/ood_datasets/dtd/images", transform=transforms.Compose([ transforms.Resize(32), transforms.CenterCrop(32), transforms.ToTensor() ])) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) elif args.out_dataset == 'places365': testsetout = torchvision.datasets.ImageFolder( root="datasets/ood_datasets/places365/test_subset", transform=transforms.Compose([ transforms.Resize(32), transforms.CenterCrop(32), transforms.ToTensor() ])) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) else: testsetout = torchvision.datasets.ImageFolder( "./datasets/ood_datasets/{}".format(args.out_dataset), transform=transform) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.batch_size, shuffle=True, num_workers=2) # set information about feature extaction temp_x = torch.rand(2, 3, 32, 32) temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) t0 = time.time() f1 = open(os.path.join(save_dir, "confidence_mahalanobis_In.txt"), 'w') f2 = open(os.path.join(save_dir, "confidence_mahalanobis_Out.txt"), 'w') N = 10000 if args.out_dataset == "iSUN": N = 8925 if args.out_dataset == "dtd": N = 5640 ########################################In-distribution########################################### print("Processing in-distribution images") if args.adv: attack = MahalanobisLinfPGDAttack(model, eps=args.epsilon, nb_iter=args.iters, eps_iter=args.iter_size, rand_init=True, clip_min=0., clip_max=1., in_distribution=True, num_classes=num_classes, sample_mean=sample_mean, precision=precision, num_output=num_output, regressor=regressor) count = 0 for j, data in enumerate(testloaderIn): images, _ = data batch_size = images.shape[0] if count + batch_size > N: images = images[:N - count] batch_size = images.shape[0] if args.adv: inputs = attack.perturb(images) else: inputs = images Mahalanobis_scores = get_Mahalanobis_score(model, inputs, num_classes, sample_mean, precision, num_output, magnitude) confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:, 1] for k in range(batch_size): f1.write("{}\n".format(-confidence_scores[k])) count += batch_size print("{:4}/{:4} images processed, {:.1f} seconds used.".format( count, N, time.time() - t0)) t0 = time.time() if count == N: break ###################################Out-of-Distributions##################################### t0 = time.time() print("Processing out-of-distribution images") if args.adv: attack = MahalanobisLinfPGDAttack(model, eps=args.epsilon, nb_iter=args.iters, eps_iter=args.iter_size, rand_init=True, clip_min=0., clip_max=1., in_distribution=False, num_classes=num_classes, sample_mean=sample_mean, precision=precision, num_output=num_output, regressor=regressor) count = 0 for j, data in enumerate(testloaderOut): images, labels = data batch_size = images.shape[0] if args.adv: inputs = attack.perturb(images) else: inputs = images Mahalanobis_scores = get_Mahalanobis_score(model, inputs, num_classes, sample_mean, precision, num_output, magnitude) confidence_scores = regressor.predict_proba(Mahalanobis_scores)[:, 1] for k in range(batch_size): f2.write("{}\n".format(-confidence_scores[k])) count += batch_size print("{:4}/{:4} images processed, {:.1f} seconds used.".format( count, N, time.time() - t0)) t0 = time.time() if count == N: break f1.close() f2.close() results = metric(save_dir, stypes) print_results(results, stypes) return
def item(report_id, want_object=False): result = (db.session.query(Report, OpSysComponent) .join(OpSysComponent) .filter(Report.id == report_id) .first()) if result is None: abort(404) report, component = result solutions = None if report.max_certainty is not None: osr = get_report_opsysrelease(db=db, report_id=report.id) solutions = [find_solution(report, db=db, osr=osr)] releases = (db.session.query(ReportOpSysRelease, ReportOpSysRelease.count) .filter(ReportOpSysRelease.report_id == report_id) .order_by(desc(ReportOpSysRelease.count)) .all()) arches = (db.session.query(ReportArch, ReportArch.count) .filter(ReportArch.report_id == report_id) .order_by(desc(ReportArch.count)) .all()) modes = (db.session.query(ReportSelinuxMode, ReportSelinuxMode.count) .filter(ReportSelinuxMode.report_id == report_id) .order_by(desc(ReportSelinuxMode.count)) .all()) history_select = lambda table, date, date_range: (db.session.query(table). filter(table.report_id == report_id) .filter(date >= date_range) # Flot is confused if not ordered .order_by(date) .all()) MAX_DAYS = 20 # Default set on 20 MAX_WEEK = 20 # Default set on 20 MAX_MONTH = 20 # Default set on 20 today = datetime.date.today() # Show only 20 days daily_history = history_select(ReportHistoryDaily, ReportHistoryDaily.day, (today - timedelta(days=MAX_DAYS))) if len(daily_history) == 0: for x in range(0, MAX_DAYS): daily_history.append({'day': today - timedelta(x), 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) elif len(daily_history) < MAX_DAYS: if daily_history[-1].day < (today): daily_history.append({'day': today, 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id }) if daily_history[0].day > (today - timedelta(MAX_DAYS)): daily_history.append({'day': today - timedelta(MAX_DAYS), 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id }) # Show only 20 weeks last_monday = datetime.datetime.today() - timedelta(datetime.datetime.today().weekday()) weekly_history = history_select(ReportHistoryWeekly, ReportHistoryWeekly.week, (last_monday - timedelta(days=MAX_WEEK*7))) if len(weekly_history) == 0: for x in range(0, MAX_WEEK): weekly_history.append({'week': last_monday - timedelta(x*7), 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) elif len(weekly_history) < MAX_WEEK: if weekly_history[-1].week < (last_monday.date()): weekly_history.append({'week': last_monday, 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) if weekly_history[0].week > ((last_monday - timedelta(7*MAX_WEEK)).date()): weekly_history.append({'week': last_monday - timedelta(7*MAX_WEEK), 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) # Show only 20 months monthly_history = history_select(ReportHistoryMonthly, ReportHistoryMonthly.month, (today - relativedelta(months=MAX_MONTH))) first_day_of_month = lambda t: (datetime.date(t.year, t.month, 1)) fdom = first_day_of_month(datetime.datetime.today()) if len(monthly_history) == 0: for x in range(0, MAX_MONTH): monthly_history.append({'month': fdom - relativedelta(months=x), 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) elif len(monthly_history) < MAX_MONTH: if monthly_history[-1].month < (fdom): monthly_history.append({'month': fdom, 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) if monthly_history[0].month > (fdom - relativedelta(months=MAX_MONTH)): monthly_history.append({'month': fdom - relativedelta(months=MAX_MONTH), 'count': 0, 'opsysrelease_id': releases[0].ReportOpSysRelease.opsysrelease_id}) complete_history = history_select(ReportHistoryMonthly, ReportHistoryMonthly.month, (datetime.datetime.strptime('1970-01-01', '%Y-%m-%d'))) unique_ocurrence_os = {} if len(complete_history) > 0: for ch in complete_history: os_name = "{0} {1}".format(ch.opsysrelease.opsys.name, ch.opsysrelease.version) if ch.count is None: ch.count = 0 if ch.unique is None: ch.count = 0 if os_name not in unique_ocurrence_os: unique_ocurrence_os[os_name] = {'count': ch.count, 'unique': ch.unique} else: unique_ocurrence_os[os_name]['count'] += ch.count unique_ocurrence_os[os_name]['unique'] += ch.unique sorted(unique_ocurrence_os) packages = load_packages(db, report_id) # creates a package_counts list with this structure: # [(package name, count, [(package version, count in the version)])] names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)}) for pkg in packages: names[pkg.iname]["name"] = pkg.iname names[pkg.iname]["count"] += pkg.count names[pkg.iname]["versions"]["{0}:{1}-{2}" .format(pkg.iepoch, pkg.iversion, pkg.irelease)] += pkg.count package_counts = [] for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True): package_counts.append(( pkg["name"], pkg["count"], sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True))) try: backtrace = report.backtraces[0].frames except: backtrace = [] fid = 0 for frame in backtrace: fid += 1 frame.nice_order = fid is_maintainer = is_component_maintainer(db, g.user, component) contact_emails = [] if is_maintainer: contact_emails = [email_address for (email_address, ) in (db.session.query(ContactEmail.email_address) .join(ReportContactEmail) .filter(ReportContactEmail.report == report))] maintainer = (db.session.query(AssociatePeople) .join(OpSysReleaseComponentAssociate) .join(OpSysReleaseComponent) .join(OpSysComponent) .filter(OpSysComponent.name == component.name)).first() maintainer_contact = "" if maintainer: maintainer_contact = maintainer.name probably_fixed = (db.session.query(ProblemOpSysRelease, Build) .join(Problem) .join(Report) .join(Build) .filter(Report.id == report_id) .first()) forward = dict(report=report, probably_fixed=probably_fixed, component=component, releases=metric(releases), arches=metric(arches), modes=metric(modes), daily_history=daily_history, weekly_history=weekly_history, monthly_history=monthly_history, complete_history=complete_history, unique_ocurrence_os=unique_ocurrence_os, crashed_packages=packages, package_counts=package_counts, backtrace=backtrace, contact_emails=contact_emails, solutions=solutions, maintainer_contact=maintainer_contact) forward['error_name'] = report.error_name forward['oops'] = report.oops if want_object: try: cf = component.name if len(report.backtraces[0].crash_function) > 0: cf += " in {0}".format(report.backtraces[0].crash_function) forward['crash_function'] = cf except: forward['crash_function'] = "" if probably_fixed: tmp_dict = probably_fixed.ProblemOpSysRelease.serialize tmp_dict['probable_fix_build'] = probably_fixed.Build.serialize forward['probably_fixed'] = tmp_dict # Avg count occurrence from first to last occurence forward['avg_count_per_month'] = get_avg_count(report.first_occurrence, report.last_occurrence, report.count) if len(forward['report'].bugs) > 0: forward['bugs'] = [] for bug in forward['report'].bugs: try: forward['bugs'].append(bug.serialize) except: print "Bug serialize failed" return forward if request_wants_json(): return jsonify(forward) forward["is_maintainer"] = is_maintainer forward["extfafs"] = get_external_faf_instances(db) return render_template("reports/item.html", **forward)
def item(report_id): result = (db.session.query(Report, OpSysComponent) .join(OpSysComponent) .filter(Report.id == report_id) .first()) if result is None: abort(404) report, component = result releases = (db.session.query(ReportOpSysRelease, ReportOpSysRelease.count) .filter(ReportOpSysRelease.report_id == report_id) .order_by(desc(ReportOpSysRelease.count)) .all()) arches = (db.session.query(ReportArch, ReportArch.count) .filter(ReportArch.report_id == report_id) .order_by(desc(ReportArch.count)) .all()) modes = (db.session.query(ReportSelinuxMode, ReportSelinuxMode.count) .filter(ReportSelinuxMode.report_id == report_id) .order_by(desc(ReportSelinuxMode.count)) .all()) history_select = lambda table: (db.session.query(table). filter(table.report_id == report_id) .all()) daily_history = history_select(ReportHistoryDaily) weekly_history = history_select(ReportHistoryWeekly) monthly_history = history_select(ReportHistoryMonthly) packages = load_packages(db, report_id, "CRASHED") related_packages = load_packages(db, report_id, "RELATED") related_packages_nevr = sorted( [metric_tuple(name="{0}-{1}:{2}-{3}".format( pkg.iname, pkg.iepoch, pkg.iversion, pkg.irelease), count=pkg.count) for pkg in related_packages], key=itemgetter(0)) merged_name = dict() for package in related_packages: if package.iname in merged_name: merged_name[package.iname] += package.count else: merged_name[package.iname] = package.count related_packages_name = sorted([metric_tuple(name=item[0], count=item[1]) for item in merged_name.items()], key=itemgetter(0), reverse=True) try: backtrace = report.backtraces[0].frames except: backtrace = [] fid = 0 for frame in backtrace: fid += 1 frame.nice_order = fid forward = dict(report=report, component=component, releases=metric(releases), arches=metric(arches), modes=metric(modes), daily_history=daily_history, weekly_history=weekly_history, monthly_history=monthly_history, crashed_packages=packages, related_packages_nevr=related_packages_nevr, related_packages_name=related_packages_name, backtrace=backtrace) if request_wants_json(): return jsonify(forward) return render_template("reports/item.html", **forward)
def item(report_id): result = (db.session.query(Report, OpSysComponent) .join(OpSysComponent) .filter(Report.id == report_id) .first()) if result is None: abort(404) report, component = result releases = (db.session.query(ReportOpSysRelease, ReportOpSysRelease.count) .filter(ReportOpSysRelease.report_id == report_id) .order_by(desc(ReportOpSysRelease.count)) .all()) arches = (db.session.query(ReportArch, ReportArch.count) .filter(ReportArch.report_id == report_id) .order_by(desc(ReportArch.count)) .all()) modes = (db.session.query(ReportSelinuxMode, ReportSelinuxMode.count) .filter(ReportSelinuxMode.report_id == report_id) .order_by(desc(ReportSelinuxMode.count)) .all()) history_select = lambda table, date: (db.session.query(table). filter(table.report_id == report_id) # Flot is confused if not ordered .order_by(date) .all()) daily_history = history_select(ReportHistoryDaily, ReportHistoryDaily.day) weekly_history = history_select(ReportHistoryWeekly, ReportHistoryWeekly.week) monthly_history = history_select(ReportHistoryMonthly, ReportHistoryMonthly.month) packages = load_packages(db, report_id) # creates a package_counts list with this structure: # [(package name, count, [(package version, count in the version)])] names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)}) for pkg in packages: names[pkg.iname]["name"] = pkg.iname names[pkg.iname]["count"] += pkg.count names[pkg.iname]["versions"]["{0}:{1}-{2}" .format(pkg.iepoch, pkg.iversion, pkg.irelease)] += pkg.count package_counts = [] for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True): package_counts.append(( pkg["name"], pkg["count"], sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True))) try: backtrace = report.backtraces[0].frames except: backtrace = [] fid = 0 for frame in backtrace: fid += 1 frame.nice_order = fid is_maintainer = is_component_maintainer(db, g.user, component) contact_emails = [] if is_maintainer: contact_emails = [email_address for (email_address, ) in (db.session.query(ContactEmail.email_address) .join(ReportContactEmail) .filter(ReportContactEmail.report == report))] forward = dict(report=report, component=component, releases=metric(releases), arches=metric(arches), modes=metric(modes), daily_history=daily_history, weekly_history=weekly_history, monthly_history=monthly_history, crashed_packages=packages, package_counts=package_counts, backtrace=backtrace, contact_emails=contact_emails) if request_wants_json(): return jsonify(forward) forward["is_maintainer"] = is_maintainer forward["extfafs"] = get_external_faf_instances(db) return render_template("reports/item.html", **forward)
testPred = [] num_batch = math.ceil(num_test / args.batch_size) start_test = time.time() for batch_idx in range(num_batch): start_idx = batch_idx * args.batch_size end_idx = min(num_test, (batch_idx + 1) * args.batch_size) feed_dict = { X: testX[start_idx:end_idx], TE: testTE[start_idx:end_idx], is_training: False } pred_batch = sess.run(pred, feed_dict=feed_dict) testPred.append(pred_batch) end_test = time.time() testPred = np.concatenate(testPred, axis=0) train_mae, train_rmse, train_mape = utils.metric(trainPred, trainY) val_mae, val_rmse, val_mape = utils.metric(valPred, valY) test_mae, test_rmse, test_mape = utils.metric(testPred, testY) utils.log_string(log, 'testing time: %.1fs' % (end_test - start_test)) utils.log_string(log, ' MAE\t\tRMSE\t\tMAPE') utils.log_string( log, 'train %.2f\t\t%.2f\t\t%.2f%%' % (train_mae, train_rmse, train_mape * 100)) utils.log_string( log, 'val %.2f\t\t%.2f\t\t%.2f%%' % (val_mae, val_rmse, val_mape * 100)) utils.log_string( log, 'test %.2f\t\t%.2f\t\t%.2f%%' % (test_mae, test_rmse, test_mape * 100)) utils.log_string(log, 'performance in each prediction step') MAE, RMSE, MAPE = [], [], []
def item(problem_id): problem = db.session.query(Problem).filter( Problem.id == problem_id).first() if problem is None: raise abort(404) report_ids = [report.id for report in problem.reports] sub = (db.session.query(ReportOpSysRelease.opsysrelease_id, func.sum(ReportOpSysRelease.count).label("cnt")) .join(Report) .filter(Report.id.in_(report_ids)) .group_by(ReportOpSysRelease.opsysrelease_id) .subquery()) osreleases = (db.session.query(OpSysRelease, sub.c.cnt) .join(sub) .order_by(desc("cnt")) .all()) sub = (db.session.query(ReportArch.arch_id, func.sum(ReportArch.count).label("cnt")) .join(Report) .filter(Report.id.in_(report_ids)) .group_by(ReportArch.arch_id) .subquery()) arches = (db.session.query(Arch, sub.c.cnt).join(sub) .order_by(desc("cnt")) .all()) exes = (db.session.query(ReportExecutable.path, func.sum(ReportExecutable.count).label("cnt")) .join(Report) .filter(Report.id.in_(report_ids)) .group_by(ReportExecutable.path) .order_by(desc("cnt")) .all()) sub = (db.session.query(ReportPackage.installed_package_id, func.sum(ReportPackage.count).label("cnt")) .join(Report) .filter(Report.id.in_(report_ids)) .group_by(ReportPackage.installed_package_id) .subquery()) packages_known = db.session.query(Package, sub.c.cnt).join(sub).all() packages_unknown = (db.session.query(ReportUnknownPackage, ReportUnknownPackage.count) .join(Report) .filter(Report.id.in_(report_ids))).all() packages = packages_known + packages_unknown # creates a package_counts list with this structure: # [(package name, count, [(package version, count in the version)])] names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)}) for (pkg, cnt) in packages: names[pkg.name]["name"] = pkg.name names[pkg.name]["count"] += cnt names[pkg.name]["versions"][pkg.evr()] += cnt package_counts = [] for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True): package_counts.append(( pkg["name"], pkg["count"], sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True))) for report in problem.reports: for backtrace in report.backtraces: fid = 0 for frame in backtrace.frames: fid += 1 frame.nice_order = fid bt_hashes = (db.session.query(ReportHash.hash) .join(Report) .join(Problem) .filter(Problem.id == problem_id) .distinct(ReportHash.hash).all()) # Limit to 10 bt_hashes (otherwise the URL can get too long) # Select the 10 hashes uniformly from the entire list to make sure it is a # good representation. (Slicing the 10 first could mean the 10 oldest # are selected which is not a good representation.) k = min(len(bt_hashes), 10) a = 0 d = len(bt_hashes)/float(k) bt_hashes_limited = [] for i in range(k): bt_hashes_limited.append("bth=" + bt_hashes[int(a)][0]) a += d bt_hash_qs = "&".join(bt_hashes_limited) forward = {"problem": problem, "osreleases": metric(osreleases), "arches": metric(arches), "exes": metric(exes), "package_counts": package_counts, "bt_hash_qs": bt_hash_qs } if request_wants_json(): return jsonify(forward) is_maintainer = is_problem_maintainer(db, g.user, problem) forward["is_maintainer"] = is_maintainer forward["extfafs"] = get_external_faf_instances(db) if report_ids: bt_diff_form = BacktraceDiffForm() bt_diff_form.lhs.choices = [(id, id) for id in report_ids] bt_diff_form.rhs.choices = bt_diff_form.lhs.choices forward['bt_diff_form'] = bt_diff_form return render_template("problems/item.html", **forward)
def train(config, bert_config, train_path, dev_path, rel2id, id2rel, tokenizer): if os.path.exists(config.output_dir) is False: os.makedirs(config.output_dir, exist_ok=True) if os.path.exists('./data/train_file.pkl'): train_data = pickle.load(open("./data/train_file.pkl", mode='rb')) else: train_data = data.load_data(train_path, tokenizer, rel2id, num_rels) pickle.dump(train_data, open("./data/train_file.pkl", mode='wb')) dev_data = json.load(open(dev_path)) for sent in dev_data: data.to_tuple(sent) data_manager = data.SPO(train_data) train_sampler = RandomSampler(data_manager) train_data_loader = DataLoader(data_manager, sampler=train_sampler, batch_size=config.batch_size, drop_last=True) num_train_steps = int( len(data_manager) / config.batch_size) * config.max_epoch if config.bert_pretrained_model is not None: logger.info('load bert weight') Bert_model = BertModel.from_pretrained(config.bert_pretrained_model, config=bert_config) else: logger.info('random initialize bert model') Bert_model = BertModel(config=bert_config).init_weights() Bert_model.to(device) submodel = sub_model(config).to(device) objmodel = obj_model(config).to(device) loss_fuc = nn.BCELoss(reduction='none') params = list(Bert_model.parameters()) + list( submodel.parameters()) + list(objmodel.parameters()) optimizer = AdamW(params, lr=config.lr) # Train! logger.info("***** Running training *****") logger.info(" Num examples = %d", len(data_manager)) logger.info(" Num Epochs = %d", config.max_epoch) logger.info(" Total train batch size = %d", config.batch_size) logger.info(" Total optimization steps = %d", num_train_steps) logger.info(" Logging steps = %d", config.print_freq) logger.info(" Save steps = %d", config.save_freq) global_step = 0 Bert_model.train() submodel.train() objmodel.train() for _ in range(config.max_epoch): optimizer.zero_grad() epoch_itorator = tqdm(train_data_loader, disable=None) for step, batch in enumerate(epoch_itorator): batch = tuple(t.to(device) for t in batch) input_ids, segment_ids, input_masks, sub_positions, sub_heads, sub_tails, obj_heads, obj_tails = batch bert_output = Bert_model(input_ids, input_masks, segment_ids)[0] pred_sub_heads, pred_sub_tails = submodel( bert_output) # [batch_size, seq_len, 1] pred_obj_heads, pred_obj_tails = objmodel(bert_output, sub_positions) # 计算loss mask = input_masks.view(-1) # loss1 sub_heads = sub_heads.unsqueeze(-1) # [batch_szie, seq_len, 1] sub_tails = sub_tails.unsqueeze(-1) loss1_head = loss_fuc(pred_sub_heads, sub_heads).view(-1) loss1_head = torch.sum(loss1_head * mask) / torch.sum(mask) loss1_tail = loss_fuc(pred_sub_tails, sub_tails).view(-1) loss1_tail = torch.sum(loss1_tail * mask) / torch.sum(mask) loss1 = loss1_head + loss1_tail # loss2 loss2_head = loss_fuc(pred_obj_heads, obj_heads).view(-1, obj_heads.shape[-1]) loss2_head = torch.sum( loss2_head * mask.unsqueeze(-1)) / torch.sum(mask) loss2_tail = loss_fuc(pred_obj_tails, obj_tails).view(-1, obj_tails.shape[-1]) loss2_tail = torch.sum( loss2_tail * mask.unsqueeze(-1)) / torch.sum(mask) loss2 = loss2_head + loss2_tail # optimize loss = loss1 + loss2 loss.backward() optimizer.step() optimizer.zero_grad() global_step += 1 if (global_step + 1) % config.print_freq == 0: logger.info( "epoch : {} step: {} #### loss1: {} loss2: {}".format( _, global_step + 1, loss1.cpu().item(), loss2.cpu().item())) if (global_step + 1) % config.eval_freq == 0: logger.info("***** Running evaluating *****") with torch.no_grad(): Bert_model.eval() submodel.eval() objmodel.eval() P, R, F1 = utils.metric(Bert_model, submodel, objmodel, dev_data, id2rel, tokenizer) logger.info(f'precision:{P}\nrecall:{R}\nF1:{F1}') Bert_model.train() submodel.train() objmodel.train() if (global_step + 1) % config.save_freq == 0: # Save a trained model model_name = "pytorch_model_%d" % (global_step + 1) output_model_file = os.path.join(config.output_dir, model_name) state = { 'bert_state_dict': Bert_model.state_dict(), 'subject_state_dict': submodel.state_dict(), 'object_state_dict': objmodel.state_dict(), } torch.save(state, output_model_file) model_name = "pytorch_model_last" output_model_file = os.path.join(config.output_dir, model_name) state = { 'bert_state_dict': Bert_model.state_dict(), 'subject_state_dict': submodel.state_dict(), 'object_state_dict': objmodel.state_dict(), } torch.save(state, output_model_file)
def item(report_id): result = (db.session.query(Report, OpSysComponent).join(OpSysComponent).filter( Report.id == report_id).first()) if result is None: abort(404) report, component = result releases = (db.session.query( ReportOpSysRelease, ReportOpSysRelease.count).filter( ReportOpSysRelease.report_id == report_id).order_by( desc(ReportOpSysRelease.count)).all()) arches = (db.session.query( ReportArch, ReportArch.count).filter(ReportArch.report_id == report_id).order_by( desc(ReportArch.count)).all()) modes = (db.session.query( ReportSelinuxMode, ReportSelinuxMode.count).filter( ReportSelinuxMode.report_id == report_id).order_by( desc(ReportSelinuxMode.count)).all()) history_select = lambda table: (db.session.query(table).filter( table.report_id == report_id).all()) daily_history = history_select(ReportHistoryDaily) weekly_history = history_select(ReportHistoryWeekly) monthly_history = history_select(ReportHistoryMonthly) packages = load_packages(db, report_id) # creates a package_counts list with this structure: # [(package name, count, [(package version, count in the version)])] names = defaultdict(lambda: {"count": 0, "versions": defaultdict(int)}) for pkg in packages: names[pkg.iname]["name"] = pkg.iname names[pkg.iname]["count"] += pkg.count names[pkg.iname]["versions"]["{0}:{1}-{2}".format( pkg.iepoch, pkg.iversion, pkg.irelease)] += pkg.count package_counts = [] for pkg in sorted(names.values(), key=itemgetter("count"), reverse=True): package_counts.append((pkg["name"], pkg["count"], sorted(pkg["versions"].items(), key=itemgetter(1), reverse=True))) try: backtrace = report.backtraces[0].frames except: backtrace = [] fid = 0 for frame in backtrace: fid += 1 frame.nice_order = fid contact_emails = [] is_maintainer = app.config["EVERYONE_IS_MAINTAINER"] if not is_maintainer and g.user is not None: if user_is_maintainer(db, g.user.username, component.id): is_maintainer = True if is_maintainer: contact_emails = [ email_address for (email_address, ) in (db.session.query( ContactEmail.email_address).join(ReportContactEmail).filter( ReportContactEmail.report == report)) ] forward = dict(report=report, component=component, releases=metric(releases), arches=metric(arches), modes=metric(modes), daily_history=daily_history, weekly_history=weekly_history, monthly_history=monthly_history, crashed_packages=packages, package_counts=package_counts, backtrace=backtrace, contact_emails=contact_emails) if request_wants_json(): return jsonify(forward) forward["is_maintainer"] = is_maintainer forward["extfafs"] = get_external_faf_instances(db) return render_template("reports/item.html", **forward)