def main(): model = Yolov1(split_size=7, num_boxes=2, num_classes=20).to(DEVICE) optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY) loss_fn = YoloLoss() train_dataset = VOCDataset("data/train.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR) test_dataset = VOCDataset("data/test.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR) train_loader=DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=1, pin_memory=PIN_MEMORY, shuffle=True,drop_last=True) test_loader=DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, num_workers=1, pin_memory=PIN_MEMORY, shuffle=True,drop_last=True) for epoch in range(EPOCHS): pred_boxes, target_boxes = get_bboxes(train_loader, model, iou_threshold=0.5, threshold=0.4) mAP = mean_average_precision(pred_boxes, target_boxes, iou_threshold=0.5) print(f"Train mAP:{mAP}") train_fn(train_loader, model, optimizer, loss_fn) if epoch > 99: for x, y in test_loader: x = x.to(DEVICE) for idx in range(16): bboxes = cellboxes_to_boxes(model(x)) bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4) plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes) if __name__ == "__main__": main()
def test_alpha_by_use_map(args, docs, data): br = prepare_bert_result(args.test) maxAlpha = 5 * 10 + 1 fianlAlpha = 0 maxmAP = 0 mAPList = [] for alpha in range(1, maxAlpha): alpha *= 0.1 mAPSub = 0 subList = [] for i in range(len(br)): res = br[i] gt = data[i] # rank = [] for j in range(len(res['doc_ids'])): rank.append((res['doc_ids'][j], alpha * res['doc_scores'][j] + gt['bm25_top1000_scores'][j])) rank = sorted(rank, key=lambda s: s[1], reverse=True) rank = [r[0] for r in rank] tmp = mean_average_precision(rank, gt['pos_doc_ids']) subList.append(tmp) mAPSub += tmp # if mAPSub > maxmAP: maxmAP = mAPSub fianlAlpha = alpha mAPList = subList print( f'average alpha : {fianlAlpha}, mAP : {maxmAP / len(br)}, mAPList : {mAPList}' ) return fianlAlpha
def validate(valid_loader, model, criterion, func, targets): losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() outputs = [] for i, (input, target, video_names, frame_indexes) in enumerate(valid_loader): b = target.size(0) input = input.cuda(async=True) target = target.cuda(async=True) input_var = torch.autograd.Variable(input, volatile=True) target_var = torch.autograd.Variable(target, volatile=True) # compute output output = model(input_var) output = func(output) # softmax, (bxr)x101 # mean probabilities over r sequences output = output.view(b, -1, num_classes).mean(1).squeeze(1) loss = criterion(torch.log(output), target_var) # needs logsoftmax outputs.append(output.data.cpu().numpy()) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1,5)) losses.update(loss.data[0], target.size(0)) top1.update(prec1[0], target.size(0)) top5.update(prec5[0], target.size(0)) outputs = np.concatenate(outputs, axis=0) mean_ap = mean_average_precision(outputs, targets, num_classes) return losses.avg, top1.avg, top5.avg, mean_ap
def main(): model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY) loss_fn = YoloLoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv") if config.LOAD_MODEL: load_checkpoint(config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE) print("loaded model") scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor( config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(config.DEVICE) for epoch in range(config.NUM_EPOCHS): print(f"\nEpoch [{epoch}]") #plot_couple_examples(model, test_loader, 0.6, 0.5, scaled_anchors) train_fn(train_loader, model, optimizer, loss_fn, scaler, scaled_anchors) #if config.SAVE_MODEL: # save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar") #print("On Train Eval loader:") #print("On Train loader:") #check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD) if epoch > 0 and epoch % 4 == 0: valid_fn(train_eval_loader, model, loss_fn, scaled_anchors) check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") print("\nnResuming Training\n") model.train()
def main(): model = Yolov1(split_size=7, num_boxes=2, num_classes=20).to(DEVICE) optimizer = optim.Adam( model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY ) loss_fn = YoloLoss() if LOAD_MODEL: load_checkpoint(torch.load(LOAD_MODEL_FILE), model, optimizer) train_dataset = VOCDataset( "data/100examples.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR ) test_dataset = VOCDataset( "data/test.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR ) train_loader = DataLoader( dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True ) test_loader = DataLoader( dataset=test_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True ) for epoch in range(EPOCHS): pred_boxes, target_boxes = get_bboxes( train_loader, model, iou_threshold=0.5, threshold=0.4 ) mean_avg_prec = mean_average_precision( pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint" ) print(f"Train mAP in {epoch}: {mean_avg_prec}") train_fn(train_loader, model, optimizer, loss_fn)
def main(): model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam( model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY ) loss_fn = YoloLoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv" ) if config.LOAD_MODEL: load_checkpoint( config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE ) scaled_anchors = ( torch.tensor(config.ANCHORS) * torch.tensor(config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2) ).to(config.DEVICE) for epoch in range(config.NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler, scaled_anchors) if epoch > 0 and epoch % 3 == 0: check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") model.train() if epoch > 99: for x, y in train_loader: x = x.to(DEVICE) for idx in range(8): bboxes = cellboxes_to_boxes(model(x)) bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4) plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes, idx) if __name__ == "__main__": main()
def main(): model = YOLOv2_lite(grid_size=GRID_SIZE, num_boxes=NUM_BOXES, num_classes=NUM_CLASSES).to(DEVICE) model.load_state_dict(torch.load(MODEL_PATH)) model.eval() test_dataset = YOLOVOCDataset( DATA_CSV, transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR, S=GRID_SIZE, B=NUM_BOXES, C=NUM_CLASSES, hflip_prob=0.5, ) test_loader = DataLoader( dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=True, drop_last=False, ) images, predictions, labels = get_bboxes( test_loader, model, iou_threshold=0.5, prob_threshold=0.4, S=GRID_SIZE, C=NUM_CLASSES, mode="all", return_images=True, ) mAP = mean_average_precision(predictions, labels, num_classes=NUM_CLASSES) print("Mean Average Precision: %.3f" % (mAP)) img_to_plot = np.random.randint(0, len(images)) image = images[img_to_plot] img_preds = [pred[1:] for pred in predictions if pred[0] == img_to_plot] img_labels = [label[1:] for label in labels if label[0] == img_to_plot] #plot_detections(image, img_preds) plot_detections(image, img_labels)
def main(): model = Yolo(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY) lossfunction = Yololoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv") if config.LOAD_MODEL: load_checkpoint(config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE) scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor( config.GRID_SIZE).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to( config.DEVICE) for epoch in range(config.NUM_EPOCHS): train(train_loader, model, optimizer, lossfunction, scaler, scaled_anchors) if config.SAVE_MODEL: save_checkpoint(model, optimizer, filename=config.CHECKPOINT_FILE) if epoch > 0 and epoch % 3 == 0: check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") model.train()
def main(): model = YOLOv1(split_size=7, num_boxes=2, num_classes=20).to(device) optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=wd) loss_fn = YoloLoss() if load_model: load_checkpoint(torch.load(load_model_file), model, optimizer) train_dataset = VOCDataset("data/8examples.csv", transform=transform, img_dir=img_dir, label_dir=label_dir) test_dataset = VOCDataset("data/test.csv", transform=transform, img_dir=img_dir, label_dir=label_dir) train_loader = DataLoader(dataset=train_dataset, batch_size=bs, num_workers=num_workers, pin_memory=pin_mem, shuffle=True, drop_last=False) test_loader = DataLoader(dataset=test_dataset, batch_size=bs, num_workers=num_workers, pin_memory=pin_mem, shuffle=True, drop_last=True) for epoch in range(epochs): pred_boxes, target_boxes = get_bboxes(train_loader, model, iou_threshold=0.5, threshold=0.4) mean_avg_prec = mean_average_precision(pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint") print(f"Train mAP: {mean_avg_prec}") train_fn(train_loader, model, optimizer, loss_fn)
def main(): model = Yolov1(split_size=7, num_boxes=2, num_classes=20).to(DEVICE) optimizer = optim.Adam( model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY ) loss_fn = YoloLoss() if LOAD_MODEL: load_checkpoint(torch.load(LOAD_MODEL_FILE), model, optimizer) train_dataset = VOCDataset( # test.csv, 8examples.csv, 100examples.csv "data/8examples.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR, ) test_dataset = VOCDataset( "data/test.csv" , transform = transform, img_dir = IMG_DIR, label_dir = LABEL_DIR ) train_loader = DataLoader( dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=False, ) test_loader = DataLoader( dataset=test_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=False, ) for epoch in range(EPOCHS): # for x, y in train_loader: # x = x.to(DEVICE) # for idx in range(8): # bboxes = cellboxes_to_boxes(model(x)) # bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4, box_format="midpoint") # plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes) # import sys # sys.exit() pred_boxes, target_boxes = get_bboxes( train_loader, model, iou_threshold=0.5, threshold=0.4 , device= DEVICE, ) mean_avg_prec = mean_average_precision( pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint" ) if mean_avg_prec > 0.9: checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint, filename=LOAD_MODEL_FILE) import time time.sleep(10) print(f"Train mAP: {mean_avg_prec}") train_fn(train_loader, model, optimizer, loss_fn)
def main(): """ Setup the model, loss function, data loader. Run the train function for each epoch. We may want to consider tune the number for non max suppression and mean average precision in order to remove false positive. BY false positive, they are a set of wrong output boxes. We can get rid of them by increasing the parameters. """ model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY) loss_fn = YoloLoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv") if config.LOAD_MODEL: load_checkpoint(config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE) scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor( config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(config.DEVICE) for epoch in range(config.NUM_EPOCHS): # plot_couple_examples(model, test_loader, 0.6, 0.5, scaled_anchors) train_fn(train_loader, model, optimizer, loss_fn, scaler, scaled_anchors) if config.SAVE_MODEL: save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar") # print(f"Currently epoch {epoch}") # print("On Train Eval loader:") # print("On Train loader:") # check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD) if epoch > 0 and epoch % 3 == 0: check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") model.train()
def _eval_model(self, model: Model, writer: SummaryWriter, step, t, eval_title, results_dict): training = model.training model.eval() if t in self.config['schedule_simple']: t_idx = self.config['schedule_simple'].index(t) else: t_idx = len(self.config['schedule_simple']) - 1 # for calculating total performance targets_total = [] probs_total = [] # Accuracy of each subset for order_i, t_i in enumerate(self.config['schedule_simple'][:t_idx + 1]): subset_name = t_i last_id = self.config['schedule_simple'][ -1] # XXX should be -1. -2 for debugging. subset = self.subsets[t_i] data = DataLoader( subset, batch_size=self.config['eval_batch_size'], num_workers=self.config['eval_num_workers'], collate_fn=self.collate_fn, ) # results is dict. {method: group_averagemeter_object} results, targets, probs = validate(subset_name, model, data, self.category_map, results_dict, last_id, self.split_cats_dict) targets_total.append(targets) probs_total.append(probs) if subset_name in results_dict: results_dict[subset_name].append(results) else: results_dict[subset_name] = [results] for metric in results.keys(): results[metric].write_to_excel( os.path.join(writer.logdir, 'results_{}.xlsx'.format(metric)), sheet_name='task {}'.format(subset_name), column_name='task {}'.format( self.config['schedule_simple'][t_idx]), info='avg') # ================================================================================================================= # calculate scores for trained tasks. prefix = 'tally_' # prefix for tensorboard plotting and csv filename targets_total = torch.cat(targets_total, axis=0) probs_total = torch.cat(probs_total, axis=0) predicts_total = probs_total > 0.5 # BCE style predicts total_metric = ['CP', 'CR', 'CF1', 'OP', 'OR', 'OF1', 'mAP'] results = dict() # reset results CP, CR, CF1, OP, OR, OF1, mAP = (AverageMeter() for _ in range(len(total_metric))) ncats = targets_total.sum(axis=0) # ignore classes in future tasks cats_in_task_idx = ncats > 0 cats_in_task_name = self.category_map[cats_in_task_idx].tolist() targets_total = targets_total probs_total = probs_total predicts_total = predicts_total # calculate score precision_pc = torch.mean( precision_score_per_class(targets_total[:, cats_in_task_idx], predicts_total[:, cats_in_task_idx], zero_division=0)) recall_pc = torch.mean( recall_score_per_class(targets_total[:, cats_in_task_idx], predicts_total[:, cats_in_task_idx], zero_division=0)) # CF1. note that CF1 is not a mean value of categories' f1_score f1_pc = ((2 * precision_pc * recall_pc) / (precision_pc + recall_pc) ) if (precision_pc + recall_pc) > 0 else torch.tensor([0.]) precision_oa = precision_score_overall( targets_total[:, cats_in_task_idx], predicts_total[:, cats_in_task_idx], zero_division=0) recall_oa = recall_score_overall(targets_total[:, cats_in_task_idx], predicts_total[:, cats_in_task_idx], zero_division=0) f1_oa = f1_score_overall(targets_total[:, cats_in_task_idx], predicts_total[:, cats_in_task_idx], zero_division=0) map_ = mean_average_precision(targets_total[:, cats_in_task_idx], probs_total[:, cats_in_task_idx]) # save to AverageMeter CP.update(precision_pc.item()) CR.update(recall_pc.item()) CF1.update(f1_pc.item()) OP.update(precision_oa.item()) OR.update(recall_oa.item()) OF1.update(f1_oa.item()) mAP.update(map_.item()) results[prefix + 'CP'] = CP results[prefix + 'CR'] = CR results[prefix + 'CF1'] = CF1 results[prefix + 'OP'] = OP results[prefix + 'OR'] = OR results[prefix + 'OF1'] = OF1 results[prefix + 'mAP'] = mAP # for reporting major, moderate, minor cateogory performances for report_name in self.split_cats_dict.keys(): reporter = Group_AverageMeter() # get report category idxes all_cats = self.category_map.tolist() task_cats = set(cats_in_task_name) report_cats = task_cats & set(self.split_cats_dict[report_name]) report_cats_idx = torch.tensor( [all_cats.index(cat) for cat in report_cats], dtype=torch.long) # CP, CR, CF1 performance of report_categories. _class_precision = precision_score_per_class( targets_total[:, report_cats_idx], predicts_total[:, report_cats_idx], zero_division=0) _class_recall = recall_score_per_class( targets_total[:, report_cats_idx], predicts_total[:, report_cats_idx], zero_division=0) _class_precision = torch.mean(_class_precision) _class_recall = torch.mean(_class_recall) # CF1 bias. note that CF1 is not a mean value of categories' f1_score _class_f1 = ((2*_class_precision*_class_recall)/(_class_precision+_class_recall)) \ if (_class_precision+_class_recall)>0 else torch.tensor([0.]) # OP, OR, OF1 performance of report_categories. _overall_precision = precision_score_overall( targets_total[:, report_cats_idx], predicts_total[:, report_cats_idx], zero_division=0) _overall_recall = recall_score_overall( targets_total[:, report_cats_idx], predicts_total[:, report_cats_idx], zero_division=0) _overall_f1 = f1_score_overall(targets_total[:, report_cats_idx], predicts_total[:, report_cats_idx], zero_division=0) # mAP performance of report_categories. _mAP = mean_average_precision(targets_total[:, report_cats_idx], probs_total[:, report_cats_idx]) reporter.update(['CP'], [_class_precision.item()], [1]) reporter.update(['CR'], [_class_recall.item()], [1]) reporter.update(['CF1'], [_class_f1.item()], [1]) reporter.update(['OP'], [_overall_precision.item()], [1]) reporter.update(['OR'], [_overall_recall.item()], [1]) reporter.update(['OF1'], [_overall_f1.item()], [1]) reporter.update(['mAP'], [_mAP.item()], [1]) reporter.total.reset() results[prefix + report_name] = reporter # write to tensorboard and csv. task_len = t_idx + 1 for metric in results.keys(): if not metric in [ prefix + 'CP', prefix + 'CR', prefix + 'OP', prefix + 'OR' ]: results[metric].write( writer, '%s/%s/%s/task_len(%d)' % (metric, eval_title, self.name, task_len), step, info='avg') results[metric].write_to_excel( os.path.join(writer.logdir, 'results_{}.xlsx'.format(metric)), sheet_name=prefix, column_name='task {}'.format( self.config['schedule_simple'][t_idx]), info='avg') # ================================================================================================================= # print performances at the end if t_idx == len(self.config['schedule_simple']) - 1: src = writer.logdir csv_files = ['major', 'moderate', 'minor', 'OF1', 'CF1', 'mAP', \ prefix+'major', prefix+'moderate', prefix+'minor', prefix+'CF1', prefix+'OF1', prefix+'mAP', \ 'forget'] for csv_file in csv_files: try: csv = pd.read_csv(os.path.join( src, 'results_{}.csv'.format(csv_file)), index_col=0) # print performance after training last task pd.set_option('display.max_rows', None) print( colorful.bold_green( '\n{:10} result'.format(csv_file)).styled_string) print(csv.round(4).iloc[:, -1]) # save as txt with open(os.path.join(src, 'summary.txt'), 'a') as summary_txt: summary_txt.write('\n') summary_txt.write('{:10} result\n'.format(csv_file)) summary_txt.write(csv.round(4).iloc[:, -1].to_string()) summary_txt.write('\n') except FileNotFoundError: print("This excperiment doesn't have {} file!! continue.". format(csv_file)) continue model.train(training) return results_dict
def main(): # Load dataset manager with open('training_set_list.pickle', 'rb') as handle: training_set = pickle.load(handle) with open('validation_set_list.pickle', 'rb') as handle: validation_set = pickle.load(handle) with open('test_set_list.pickle', 'rb') as handle: test_set = pickle.load(handle) with open('genres.json') as json_data: genres = json.load(json_data) with open('labels.json') as json_data: labels = json.load(json_data) log_file_name = str(datetime.now()) + '-logs.txt' with open("logs/" + log_file_name, 'w') as log_file: log_file.write('Training logs \n') iteration_file_name = str(datetime.now()) + '-iteration.txt' with open("logs/" + iteration_file_name, 'w') as log_file: log_file.write('Training iterations \n') best_iteration_file_name = str(datetime.now()) + '-best_iteration.txt' with open("logs/" + best_iteration_file_name, 'w') as log_file: log_file.write('Best Training iterations \n') dataset_manager = DatasetManager(training_set, validation_set, test_set, genres, labels) # Learning params learning_rate = 0.001 batch_size = 50 # Nombre d'iterations training_iters = 1000 # display training information (loss, training accuracy, ...) every 10 # iterations local_train_step = 20 global_validation_step = 25 # test every global_test_step iterations global_train_step = 100 # Network params n_classes = 26 keep_rate = 0.5 # for dropout # Graph input x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, n_classes]) keep_var = tf.placeholder(tf.float32) # Model pred = Model.alexnet(x, keep_var) # definition of the network architecture # Loss and optimizer # loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=pred, labels=y)) loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(y, pred)))) # optimizer = tf.train.GradientDescentOptimizer( # learning_rate=learning_rate).minimize(loss) optimizer = tf.train.RMSPropOptimizer( learning_rate=learning_rate).minimize(loss) # Init init = tf.global_variables_initializer() # Initialize an saver for store model checkpoints saver = tf.train.Saver() # To do early stopping max_validation_map = 0 # Launch the graph with tf.Session() as sess: sess.run(init) # Load pretrained model # Skip weights from fc8 (fine-tuning) load_with_skip('pretrained_alexnet.npy', sess, ['fc8']) print('Start training.') step = 1 while step < training_iters: # print("Iter ", step) with open("logs/" + iteration_file_name, 'a') as log_file: log_file.write("Iter {} \n".format(step)) batch_xs, batch_ys = dataset_manager.next_batch( batch_size, 'train') sess.run(optimizer, feed_dict={ x: batch_xs, y: batch_ys, keep_var: keep_rate }) # Display on batch training status if step % local_train_step == 0: local_train_output = sess.run(pred, feed_dict={ x: batch_xs, keep_var: 1 }) MAP = mean_average_precision(local_train_output, batch_ys) batch_loss = sess.run(loss, feed_dict={ x: batch_xs, y: batch_ys, keep_var: 1. }) with open("logs/" + log_file_name, 'a') as log_file: log_file.write("Iter {} Training Loss = {:.4f}, " "Mean average precision = {:.4f} \n".format( step, batch_loss, MAP)) # Display global training error if step % global_train_step == 0: train_map_global = 0. test_count = 0 # test accuracy by group of batch_size images for _ in range( int(len(dataset_manager.training_list) / batch_size) + 1): batch_tx, batch_ty = dataset_manager.next_batch( batch_size, 'train') test_output = sess.run(pred, feed_dict={ x: batch_tx, keep_var: 1 }) MAP = mean_average_precision(test_output, batch_ty) train_map_global += MAP test_count += 1 train_map_global /= test_count with open("logs/" + log_file_name, 'a') as log_file: log_file.write( "Global Training Accuracy = {:.4f} \n".format( train_map_global)) # Display global testing error if step % global_validation_step == 0: validation_map_global = 0. validation_count = 0 # test accuracy by group of batch_size images for _ in range( int(len(dataset_manager.test_list) / batch_size) + 1): batch_tx, batch_ty = dataset_manager.next_batch( batch_size, 'val') test_output = sess.run(pred, feed_dict={ x: batch_tx, keep_var: 1 }) MAP = mean_average_precision(test_output, batch_ty) validation_map_global += MAP validation_count += 1 validation_map_global /= validation_count with open("logs/" + log_file_name, 'a') as log_file: log_file.write( "Iter {} Global Validation Accuracy = {:.4f} \n". format(step, validation_map_global)) if validation_map_global >= max_validation_map: max_validation_map = validation_map_global with open("logs/" + best_iteration_file_name, 'a') as log_file: log_file.write("Iter {} \n".format(step)) # Save model saver.save( sess, "saved_models/model_dropout05_mean_square_error.ckpt") step += 1 # print("Finish!") with open("logs/finish", 'w') as finish_file: finish_file.write("Finish")
def train_loop(cfg_path, gpu_n='0'): # get configs with open(cfg_path, 'r') as stream: config = yaml.safe_load(stream) device = torch.device('cuda:{}'.format(gpu_n) if config['GPU'] and torch.cuda.is_available else 'cpu') dtype = torch.float32 # TODO: find out how it affects speed and accuracy MODEL = config['MODEL'] LOAD_MODEL = config['LOAD_MODEL'] LOAD_MODEL_FILE = config['LOAD_MODEL_FILE'] SAVE_MODEL = config['SAVE_MODEL'] SAVE_MODEL_N = config['SAVE_MODEL_N'] SAVE_MODEL_DIR = config['SAVE_MODEL_DIR'] DATASET_DIR = config['DATASET_DIR'] L_RATE = config['LEARNING_RATE'] DECAY_RATE = config['DECAY_RATE'] DECAY_EPOCHS = config['DECAY_EPOCHS'] WEIGHT_DECAY = config['WEIGHT_DECAY'] EPOCHS = config['EPOCHS'] BATCH_SIZE = config['BATCH_SIZE'] NUM_WORKERS = config['NUM_WORKERS'] PIN_MEMORY = config['PIN_MEMORY'] CSV_TRAIN = config['CSV_TRAIN'] CSV_VAL = config['CSV_VAL'] # set up model if MODEL == 'Darknet': model = YoloV1(grid_size=7, num_boxes=2, num_classes=20).to(DEVICE) elif MODEL == 'VGG': pass # add here VGG backbone if LOAD_MODEL: # TODO: load backbone # cfg_cp, start_epoch = load_checkpoint(LOAD_MODEL_FILE, model) val = input( 'Do you want to use config from checkpoint? Answer "yes" or "no": ' ) # if 'val' == 'yes': # L_RATE = cfg_cp['LEARNING_RATE'] # DECAY_RATE = cfg_cp['DECAY_RATE'] # DECAY_EPOCHS = cfg_cp['DECAY_EPOCHS'] # WEIGHT_DECAY = cfg_cp['WEIGHT_DECAY'] # BALANCED = cfg_cp['BALANCED_DATASET'] # BATCH_SIZE = cfg_cp['BATCH_SIZE'] # NUM_WORKERS = cfg_cp['NUM_WORKERS'] # PIN_MEMORY = cfg_cp['PIN_MEMORY'] # MIN_IMAGES = cfg_cp['MIN_IMAGES'] # LOSS = cfg_cp['LOSS'] else: model = init_weights(model) start_epoch = 0 optimizer = optim.Adam(model.parameters(), lr=L_RATE, weight_decay=WEIGHT_DECAY) loss_fn = YoloLoss() loader_params = BATCH_SIZE, NUM_WORKERS, PIN_MEMORY, DATASET_DIR, CSV_TRAIN, CSV_VAL loader = get_dataloader(loader_params) # create folder to save models if SAVE_MODEL: if not os.path.exists('{}/{}'.format(SAVE_MODEL_DIR, MODEL)): os.makedirs('{}/{}'.format(SAVE_MODEL_DIR, MODEL)) losses, accuracies = { 'train': [], 'validate': [] }, { 'train': [], 'validate': [] } for epoch in range(start_epoch, EPOCHS + start_epoch): t = time() if (epoch + 1) % DECAY_EPOCHS == 0: L_RATE *= (1 - DECAY_RATE) optimizer = optim.Adam(model.parameters(), lr=L_RATE, weight_decay=WEIGHT_DECAY) # print epoch number print_report(part='start', epoch=epoch) # train loop train_epoch(loader['train'], model, optimizer, device, loss_fn) # print metrics pred_bb, target_bb = get_bboxes(loader['train'], model, iou_threshold=0.5, threshold=0.4) train_map = mean_average_precision(pred_bb, target_bb, iou_threshold=0.5, box_format='midpoint') v_pred_bb, v_target_bb = get_bboxes(loader['val'], model, iou_threshold=0.5, threshold=0.4) val_map = mean_average_precision(v_pred_bb, v_target_bb, iou_threshold=0.5, box_format='midpoint') metrics = -1, -1, train_map, val_map print_report(part='accuracy', metrics=metrics) # collect metrics # losses['train'].append(train_loss) # losses['validate'].append(val_loss) # accuracies['train'].append(train_acc) # accuracies['validate'].append(val_acc) # save models # if SAVE_MODEL: # save_checkpoint(model=model, cfg=cfg, epoch=epoch, loss=round(val_loss, 3)) # print time print_report(part='end', t=int(time() - t))
def main(): # Load dataset manager with open('training_set_list.pickle', 'rb') as handle: training_set = pickle.load(handle) with open('validation_set_list.pickle', 'rb') as handle: validation_set = pickle.load(handle) with open('test_set_list.pickle', 'rb') as handle: test_set = pickle.load(handle) with open('genres.json') as json_data: genres = json.load(json_data) with open('labels.json') as json_data: labels = json.load(json_data) dataset_manager = DatasetManager(training_set, validation_set, test_set, genres, labels) batch_size = 1 n_classes = 26 # Graph input x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, n_classes]) keep_var = tf.placeholder(tf.float32) # Model pred = Model.alexnet(x, keep_var) # definition of the network architecture # Loss and optimize # Init init = tf.global_variables_initializer() # Initialize an saver for store model checkpoints saver = tf.train.Saver() # Launch the graph with tf.Session() as sess: sess.run(init) # Load pretrained model # Skip weights from fc8 (fine-tuning) # load_with_skip('pretrained_alexnet.npy', sess, ['fc8']) # saver.restore(sess, "saved_models/MSE_without_data_augmentation_0.75_0.001/film_genre_model.ckpt") saver.restore( sess, "saved_models/models/model_dropout05_mean_square_error.ckpt") print('Model Restored') test_map_global = 0. test_count = 0 # test accuracy by group of batch_size images for _ in range(int(len(dataset_manager.test_list) / batch_size) + 1): batch_tx, batch_ty = dataset_manager.next_batch(batch_size, 'test') # print(batch_tx[0], batch_ty[0]) test_output = sess.run(pred, feed_dict={x: batch_tx, keep_var: 1}) # print(test_output[0]) MAP = mean_average_precision(test_output, batch_ty) test_map_global += MAP test_count += 1 test_map_global /= test_count print("Global Test Accuracy = {:.4f}".format(test_map_global)) # Load one image img = cv2.imread('saved_models/images_tests/yellow.jpg') img = cv2.resize(img, (227, 227)) img = img.astype(np.float32) img -= np.array([104., 117., 124.]) print(img) test_output = sess.run(pred, feed_dict={ x: np.reshape(img, (1, 227, 227, 3)), keep_var: 1 }) score_dict = {} for score, genre in zip(test_output[0], genres): score_dict[genre] = score print(list(reversed(sorted(score_dict.items(), key=lambda x: x[1]))))
train_embedings, test_embedings, y_train, y_test, k=50, n_cell_types=n_cell_types) elif model_type == "base_nn": all_probs = model.predict(x_test) order = np.argsort(-all_probs, axis=-1) y_pred = order[:, 0] y_probs = np.array( [all_probs[i, order[i, 0]] for i in range(all_probs.shape[0])]) accuracy = accuracy_score(y_test, y_pred) overall_map = mean_average_precision(y_test, y_pred, y_probs) cell_type_map = get_cell_type_map(n_cell_types, y_test, y_pred, y_probs) np.savetxt("test_label", y_test) np.savetxt("pred_label", y_pred) with open(out_file, "w") as out_file: out_file.writelines('Model type: ' + model_type + "\n") out_file.writelines('Number of epochs: ' + str(epochs) + "\n") out_file.writelines('Regularization: l1 ' + str(l1_reg) + " l2 " + str(l2_reg) + "\n") out_file.writelines('Test accuracy: ' + str(accuracy) + "\n") out_file.writelines('Overall MAP: ' + str(overall_map) + '\n') out_file.writelines('Cell type specific MAP:\n') out_file.writelines('\t\t\t\t\t' + '\t'.join(label_names) + '\n') out_file.writelines('\t\t\t\t\t' + '\t'.join(map(str, cell_type_map)) + '\n')
def main(args): # scale normalized anchors by their corresponding scales (e.g img_size = 416, feature maps will 32, 16, 8, # scaled_anchors = normalized_anchors X (8, 16, 32)) """ scaled anchors tensor([[ 3.6250, 2.8125], [ 4.8750, 6.1875], [11.6562, 10.1875], [ 1.8750, 3.8125], [ 3.8750, 2.8125], [ 3.6875, 7.4375], [ 1.2500, 1.6250], [ 2.0000, 3.7500], [ 4.1250, 2.8750]]) """ di = torch.tensor([ int(args.img_size / 32), int(args.img_size / 16), int(args.img_size / 8) ]).unsqueeze(1) scaled_anchors = (torch.tensor(anchors) * torch.repeat_interleave( di, torch.tensor([3, 3, 3]), dim=0).repeat(1, 2)).to(args.device) train_dataset = YOLODataset(args.train_img_dir, args.train_label_dir, args.train_annotation_file, scaled_anchors, num_classes, args.iou_thre, args.img_size, transform=train_transforms) val_dataset = YOLODataset(args.val_img_dir, args.val_label_dir, args.val_annotation_file, scaled_anchors, num_classes, args.iou_thre, args.img_size, transform=test_transforms) train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=True) model = YOLO(num_classes).to(args.device) # if it is first time to train the model, use darknet53 pretrained weights # load_weights_darknet53("darknet53.conv.74", model) optimizer = optim.Adam(model.parameters(), lr=1e-5, weight_decay=1e-4) load_checkpoint("checkpoint.pth.tar", model, optimizer, 1e-5) for epoch in range(args.epoches): train_loop(train_loader, model, YOLOloss, optimizer, args.device, scaled_anchors) print(f"Epoch {epoch+1}\n-------------------------------") save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar") model.eval() results = check_class_accuracy(model, val_loader, args.conf_thre) mAP = mean_average_precision(model, val_loader, scaled_anchors, num_classes, args.iou_thre) results.append(mAP) with open("result.txt", 'a') as f: f.write(str(results).strip("[").strip("]") + '\n') model.train() print("Done!")
def main(): model = Yolov1(split_size=S, num_boxes=B, num_classes=C).to(DEVICE) optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY) loss_fn = YoloLoss(S=S, B=B, C=C) if LOAD_MODEL: load_checkpoint(torch.load(LOAD_MODEL_FILE), model, optimizer) train_dataset = VOCDataset( training_path= '/home/mt/Desktop/For_github/computer_vision_projects/face_recognition/data', S=3, C=2, transform=transform) # test_dataset = VOCDataset( # "data/test.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR, # ) train_loader = DataLoader( dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True, ) # test_loader = DataLoader( # dataset=test_dataset, # batch_size=BATCH_SIZE, # num_workers=NUM_WORKERS, # pin_memory=PIN_MEMORY, # shuffle=True, # drop_last=True, # ) for epoch in range(EPOCHS): # for x, y in train_loader: # x = x.to(DEVICE) # for idx in range(8): # bboxes = cellboxes_to_boxes(model(x)) # bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4, box_format="midpoint") # plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes) # import sys # sys.exit() pred_boxes, target_boxes = get_bboxes(train_loader, model, iou_threshold=0.5, threshold=0.4) mean_avg_prec = mean_average_precision(pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint") print(f"Train mAP: {mean_avg_prec}") #if mean_avg_prec > 0.9: # checkpoint = { # "state_dict": model.state_dict(), # "optimizer": optimizer.state_dict(), # } # save_checkpoint(checkpoint, filename=LOAD_MODEL_FILE) # import time # time.sleep(10) train_fn(train_loader, model, optimizer, loss_fn)
def test(data_loader, gallery_loader, distance, cuda): batch_time = LogMetric.AverageMeter() acc = LogMetric.AverageMeter() meanap = LogMetric.AverageMeter() end = time.time() distance.eval() dist_matrix = [] start = time.time() with torch.no_grad(): g_gallery = [] target_gallery = [] for j, (g, target) in enumerate(gallery_loader): if cuda: g.to(torch.device('cuda')) g.gdata['std'] = g.gdata['std'].cuda() target_gallery.append(target) g_gallery.append(g) target_gallery = np.array(np.concatenate(target_gallery)) gdata = list(map(lambda g: g.gdata['std'], g_gallery)) g_gallery = dgl.batch(g_gallery) g_gallery.gdata = {'std': torch.cat(gdata)} target_query = [] for i, (g, target) in enumerate(data_loader): # Prepare input data if cuda: g.to(torch.device('cuda')) g.gdata['std'] = g.gdata['std'].cuda() # Output d = distance(g, g_gallery, mode='retrieval') dist_matrix.append(d) target_query.append(target) dist_matrix = torch.stack(dist_matrix) target_query = np.array(np.concatenate(target_query)) target_combined_query = np.unique(target_query) combined_dist_matrix = torch.zeros(target_combined_query.shape[0], dist_matrix.shape[1]) for i, kw in enumerate(target_combined_query): ind = kw == target_query combined_dist_matrix[i] = dist_matrix[ind].min(0).values # K-NN classifier acc.update( knn_accuracy(combined_dist_matrix, target_gallery, target_combined_query, k=5, dataset=data_loader.dataset.dataset)) # mAP retrieval meanap.update( mean_average_precision(combined_dist_matrix, target_gallery, target_combined_query)) batch_time.update(time.time() - start) print( '* Test Acc {acc.avg:.3f}; mAP {meanap.avg: .3f}; Time x Test {b_time.avg:.3f}' .format(acc=acc, meanap=meanap, b_time=batch_time)) return acc, meanap
def validate(task_id, model, data, cat_map, results_dict, last_id, additional_report_cats): """ :param additional_report_cats: categories list for additional report. \ dict {report_name: [cat1, cat2, cat3, ...] \ ex) major: ['train', 'horse', 'bird', 'clock', ...], \ minor: ['bicycle', 'potted plant', ....], \ moderate: ['kite', 'bench', 'teddy bear', ...] """ results = {} losses = AverageMeter() # loss (per word decoded) accuracies = AverageMeter() class_precisions = Group_AverageMeter() class_recalls = Group_AverageMeter() class_f1s = Group_AverageMeter() overall_precisions = AverageMeter() overall_recalls = AverageMeter() overall_f1s = AverageMeter() mAP = AverageMeter() criterion = model.criterion print( colorful.bold_yellow('Validating Task: ' + str(task_id)).styled_string) cpu_targets = [] cpu_predicts = [] cpu_probs = [] with torch.no_grad(): for i, (imgs, cats) in enumerate(iter(data)): # batch_size batch_size = imgs.shape[0] # Move to device, if available imgs = imgs.to(model.device) cats = cats.to(model.device) # Forward prop. predict = model.encoder(imgs) targets = cats # Calculate loss loss = criterion(predict, targets) loss = loss.mean() predict = torch.sigmoid(predict) # for mAP score cpu_probs.append(predict.cpu()) predict = predict > 0.5 # BCE total_relevant_slots = targets.sum().data relevant_predict = (predict * targets.float()).sum().data acc = relevant_predict / total_relevant_slots losses.update(loss.item(), batch_size) accuracies.update(acc, batch_size) cpu_targets.append(targets.cpu()) cpu_predicts.append(predict.cpu()) cpu_targets = torch.cat(cpu_targets, axis=0) cpu_predicts = torch.cat(cpu_predicts, axis=0) cpu_probs = torch.cat(cpu_probs, axis=0) ncats = cpu_targets.sum(axis=0) # ignore classes in other tasks cats_in_task_idx = ncats > 0 ncats = ncats[cats_in_task_idx].tolist() f1_pc = f1_score_per_class(cpu_targets[:, cats_in_task_idx], cpu_predicts[:, cats_in_task_idx], zero_division=0) precision_pc = precision_score_per_class( cpu_targets[:, cats_in_task_idx], cpu_predicts[:, cats_in_task_idx], zero_division=0) recall_pc = recall_score_per_class(cpu_targets[:, cats_in_task_idx], cpu_predicts[:, cats_in_task_idx], zero_division=0) f1_oa = f1_score_overall(cpu_targets[:, cats_in_task_idx], cpu_predicts[:, cats_in_task_idx], zero_division=0) precision_oa = precision_score_overall(cpu_targets[:, cats_in_task_idx], cpu_predicts[:, cats_in_task_idx], zero_division=0) recall_oa = recall_score_overall(cpu_targets[:, cats_in_task_idx], cpu_predicts[:, cats_in_task_idx], zero_division=0) # record performances cats_in_task_name = cat_map[cats_in_task_idx].tolist() class_f1s.update(cats_in_task_name, f1_pc.tolist(), ncats) class_precisions.update(cats_in_task_name, precision_pc.tolist(), ncats) class_recalls.update(cats_in_task_name, recall_pc.tolist(), ncats) overall_f1s.update(f1_oa.item(), len(cpu_targets)) overall_precisions.update(precision_oa.item(), len(cpu_targets)) overall_recalls.update(recall_oa.item(), len(cpu_targets)) # mAP mAP.update( mean_average_precision(cpu_targets[:, cats_in_task_idx], cpu_probs[:, cats_in_task_idx])) # for reporting major, moderate, minor cateogory performances for report_name in additional_report_cats.keys(): reporter = Group_AverageMeter() # get report category idxes all_cats = cat_map.tolist() task_cats = set(cats_in_task_name) report_cats = task_cats & set(additional_report_cats[report_name]) cats_idx = [] for cat in report_cats: cats_idx.append(all_cats.index(cat)) report_cats_idx = torch.tensor(cats_idx, dtype=torch.long) # there are tasks where the min/mod/maj are missing. if len(report_cats_idx) == 0: reporter.update(['CP'], [float('NaN')], [1]) reporter.update(['CR'], [float('NaN')], [1]) reporter.update(['CF1'], [float('NaN')], [1]) reporter.update(['OP'], [float('NaN')], [1]) reporter.update(['OR'], [float('NaN')], [1]) reporter.update(['OF1'], [float('NaN')], [1]) reporter.update(['mAP'], [float('NaN')], [1]) # for major, moderate and minor report, total is a meaningless metric. # mean of CP, CR, CF1, ... is meaningless. reporter.total.reset() # add to results results[report_name] = reporter continue # CP, CR, CF1 performance of report_categories. _class_precision = precision_score_per_class( cpu_targets[:, report_cats_idx], cpu_predicts[:, report_cats_idx], zero_division=0) _class_recall = recall_score_per_class( cpu_targets[:, report_cats_idx], cpu_predicts[:, report_cats_idx], zero_division=0) _class_precision = torch.mean(_class_precision) _class_recall = torch.mean(_class_recall) # CF1 bias. note that CF1 is not a mean value of categories' f1_score _class_f1 = ((2*_class_precision*_class_recall)/(_class_precision+_class_recall)) \ if (_class_precision+_class_recall)>0 else torch.tensor([0.]) # OP, OR, OF1 performance of report_categories. _overall_precision = precision_score_overall( cpu_targets[:, report_cats_idx], cpu_predicts[:, report_cats_idx], zero_division=0) _overall_recall = recall_score_overall( cpu_targets[:, report_cats_idx], cpu_predicts[:, report_cats_idx], zero_division=0) _overall_f1 = f1_score_overall(cpu_targets[:, report_cats_idx], cpu_predicts[:, report_cats_idx], zero_division=0) # mAP performance of report_categories. _mAP = mean_average_precision(cpu_targets[:, report_cats_idx], cpu_probs[:, report_cats_idx]) reporter.update(['CP'], [_class_precision.item()], [1]) reporter.update(['CR'], [_class_recall.item()], [1]) reporter.update(['CF1'], [_class_f1.item()], [1]) reporter.update(['OP'], [_overall_precision.item()], [1]) reporter.update(['OR'], [_overall_recall.item()], [1]) reporter.update(['OF1'], [_overall_f1.item()], [1]) reporter.update(['mAP'], [_mAP.item()], [1]) # for major, moderate and minor report, total is a meaningless metric. # mean of CP, CR, CF1, ... is meaningless. reporter.total.reset() # add to results results[report_name] = reporter # CF1 bias. note that CF1 is not a mean value of categories' f1_score class_f1s.total.reset() p_pc, r_pc = torch.mean(precision_pc).item(), torch.mean( recall_pc).item() class_f1s.total.update(((2 * p_pc * r_pc) / (p_pc + r_pc)) if (p_pc + r_pc) > 0 else 0) # save performances results['OF1'] = overall_f1s results['OP'] = overall_precisions results['OR'] = overall_recalls results['CF1'] = class_f1s results['CP'] = class_precisions results['CR'] = class_recalls results['losses'] = losses results['accuracies'] = accuracies results['mAP'] = mAP # Forgetting Measure if int(task_id) == int(last_id) and len(results_dict) > 0: forget_metrics = ['mAP', 'OF1', 'CF1'] forget = Group_AverageMeter() Cf1_forget = Group_AverageMeter() forget_results = {} per_cat_forget_results = {} for metric in forget_metrics: per_metric_results = {} for task_name, per_task_results in results_dict.items(): if metric == 'CF1': per_total_lst = [] per_cat_dict = {} # only up to the 2nd last are used to find the max. for per_task_result in per_task_results[:-1]: per_total_lst.append( per_task_result[metric].total.avg) for cat, cat_avgmtr in per_task_result[ metric].data.items(): if cat in per_cat_dict: per_cat_dict[cat].append(cat_avgmtr.avg) else: per_cat_dict[cat] = [cat_avgmtr.avg] final_task_result = per_task_results[-1][ metric].total.avg max_task_result = max(per_total_lst) # subtract the very last added and max of the tasks before. metric_forgot = None if max_task_result == 0 and final_task_result == 0: forget_results[metric + '_' + str(task_name)] = 1.0 metric_forgot = 1.0 elif max_task_result == 0 and final_task_result != 0: metric_forgot = (max_task_result - final_task_result) / 1 forget_results[metric + '_' + str(task_name)] = metric_forgot else: metric_forgot = ( max_task_result - final_task_result) / abs(max_task_result) forget_results[metric + '_' + str(task_name)] = metric_forgot for cat, catobj in per_task_results[-1][ metric].data.items(): max_cat_result = max(per_cat_dict[cat]) final_cat_result = catobj.avg if max_cat_result == 0 and final_cat_result == 0: per_cat_forget_results[cat] = 1.0 elif max_cat_result == 0 and final_cat_result != 0: per_cat_forget_results[ cat] = max_cat_result - catobj.avg / 1 else: per_cat_forget_results[cat] = (max_cat_result \ - catobj.avg)/abs(max_cat_result) else: per_metric_lst = [] for per_task_result in per_task_results[:-1]: per_metric_lst.append(per_task_result[metric].avg) metric_forgot = None final_task_result = per_task_results[-1][metric].avg max_task_result = max(per_metric_lst) if max_task_result == 0 and final_task_result == 0: metric_forgot = 1.0 forget_results[metric + '_' + str(task_name)] = metric_forgot elif max_task_result == 0 and final_task_result != 0: metric_forgot = (max_task_result - final_task_result) / 1 forget_results[metric + '_' + str(task_name)] = metric_forgot else: metric_forgot = ( max_task_result - final_task_result) / abs(max_task_result) forget_results[metric + '_' + str(task_name)] = metric_forgot for split in ['major', 'moderate', 'minor']: # check if split results in all NaNs if math.isnan( per_task_results[0][split].data[metric].avg): forget_results[split + '_' + metric + '_' + str(task_name)] = float('NaN') continue per_metric_lst = [] for per_task_result in per_task_results[:-1]: per_metric_lst.append( per_task_result[split].data[metric].avg) final_task_result = per_task_results[-1][split].data[ metric].avg max_task_result = max(per_metric_lst) split_forgot = None if max_task_result == 0 and final_task_result == 0: split_forgot = 1.0 # forgotten within the first task by majority dominance. forget_results[split + '_' + metric + '_' + str(task_name)] = split_forgot elif max_task_result == 0 and final_task_result != 0: split_forgot = (max_task_result - final_task_result) / 1 forget_results[split + '_' + metric + '_' + str(task_name)] = split_forgot else: split_forgot = (max_task_result - final_task_result ) / abs(max_task_result) forget_results[split + '_' + metric + '_' + str(task_name)] = split_forgot if metric + split + 'Overall' in per_metric_results.keys( ): per_metric_results[metric + split + 'Overall'].append(split_forgot) else: per_metric_results[metric + split + 'Overall'] = [split_forgot] if metric + 'Overall' in per_metric_results.keys(): per_metric_results[metric + 'Overall'].append(metric_forgot) else: per_metric_results[metric + 'Overall'] = [metric_forgot] forget_results[metric + 'Overall'] = average_lst( per_metric_results[metric + 'Overall']) forget_results[metric + 'majorOverall'] = average_lst( per_metric_results[metric + 'majorOverall']) forget_results[metric + 'moderateOverall'] = average_lst( per_metric_results[metric + 'moderateOverall']) forget_results[metric + 'minorOverall'] = average_lst( per_metric_results[metric + 'minorOverall']) keys = [] values = [] n = [] for k, v in forget_results.items(): keys.append(k) values.append(v) n.append(1) forget.update(keys, values, n) keys = [] values = [] n = [] for k, v in per_cat_forget_results.items(): keys.append(k) values.append(v) n.append(1) Cf1_forget.update(keys, values, n) results['forget'] = forget results['class_forget'] = Cf1_forget print( colorful.bold_cyan( 'LOSS - {loss.avg:.3f}, ACCURACY - {acc.avg:.3f}, RECALL - {rcl.total.avg:.4f},\ PRECISION - {prc.total.avg:.4f}, F1 - {f1.total.avg:.4f}'.format( loss=losses, acc=accuracies, rcl=class_recalls, prc=class_precisions, f1=class_f1s)).styled_string) return results, cpu_targets, cpu_probs
def main(): IMG_DIR = "../datasets/voc/images" LABEL_DIR = "../datasets/voc/labels" MAPPING_FILE = "../datasets/voc/100examples.csv" BATCH_SIZE = 8 NUM_WORKERS = 2 PIN_MEMORY = True DEVICE = "cuda" if torch.cuda.is_available() else "cpu" LEARNING_RATE = 4e-4 WEIGHT_DECAY = 0.0 EPOCHS = 100 LOAD_MODEL_FILE = "retrained_grad_true.pth.tar" S = 7 C = 20 B = 2 PRETRAINED = True print("Using Device:", DEVICE) class Compose(object): def __init__(self, transforms): self.transforms = transforms def __call__(self, image, bboxes): for t in self.transforms: image = t(image) return image, bboxes # Types of preprocessing transforms we want to apply convert_transform = transforms.ToTensor() resize_transform = transforms.Resize((448, 448)) # normalize_transform = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform = Compose([convert_transform, resize_transform]) train_dataset = VOCDataset(MAPPING_FILE, S=S, C=C, B=B, transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR) train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True) val_dataset = VOCDataset(MAPPING_FILE, S=S, C=C, B=B, transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR) val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True) model = YoloV1(S=S, B=B, C=C, pretrained=PRETRAINED).to(DEVICE) for counter, param in enumerate(model.resnet.parameters()): param.requires_grad = True model.resnet.fc.requires_grad = True for counter, param in enumerate(model.parameters()): print(counter, param.requires_grad) parameters = [param.numel() for param in model.parameters()] print("Number of model parameters", sum(parameters)) parameters = [param.numel() for param in model.resnet.parameters()] print("Number of resnet parameters", sum(parameters)) parameter_list = filter(lambda p: p.requires_grad, model.parameters()) optimizer = optim.Adam(parameter_list, lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY) yolo_loss = YoloLoss(S=S, C=C, B=B) writer = SummaryWriter() current_time = time.time() mean_loss_list = [] for epoch in range(EPOCHS): mean_loss = [] pred_boxes, target_boxes = get_bboxes(train_loader, model, iou_threshold=0.5, threshold=0.4, device=DEVICE) mean_avg_prec = mean_average_precision(pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint") # Save the model if mean_avg_prec > 0.99 or epoch == (EPOCHS - 1): checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } print("=> Saving checkpoint") torch.save(checkpoint, LOAD_MODEL_FILE) time.sleep(10) for batch_idx, (x, y) in enumerate(train_loader): x, y = x.to(DEVICE), y.to(DEVICE) y_p = model(x) # y = torch.flatten(y, start_dim=1, end_dim=3) y_p = torch.reshape(y_p, (BATCH_SIZE, S, S, C + 5 * B)) loss = yolo_loss(y_p, y) mean_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() # Calculate average loss mean_loss = sum(mean_loss)/len(mean_loss) delta_time = time.time() - current_time current_time = time.time() writer.add_scalar("Average Loss: ", mean_loss, epoch) writer.add_scalar("Mean Average Precision: ", mean_avg_prec, epoch) writer.add_scalar("Epoch Duration [s]", delta_time, epoch) print("Epoch:", epoch) mean_loss_list.append(mean_loss) writer.close()