def run(self, data, gt): labels = obtain_hard_clusters(data, self.thresholds) # compute scores, save labels and scores for each threshold results_list = [] for i, threshold in enumerate(self.thresholds): np.save( os.path.join( self.target_dir, "single_linkage_labels_threshold_{}.npy".format( threshold)), labels[:, i, ...]) # compute metrics results = { "parameters": { "threshold": threshold }, "scores": compute_metrics(labels[:, i, ...], gt.copy()) } # save results np.save( os.path.join( self.target_dir, "single_linkage_scores_threshold_{}".format(threshold)), np.array([ results["scores"]["CREMI_score"], results["scores"]["arand"], results["scores"]["voi"][0], results["scores"]["voi"][1] ])) results_list.append(results) return sorted(results_list, key=lambda x: x["scores"]["CREMI_score"])[0]
def run(self, data, gt): results_list = [] for min_samples_name in tqdm(self.min_samples_list, desc="Processing min_samples", leave=False): for min_cluster_size in tqdm(self.min_clusters_size_list, desc="Processing min_cluster_size", leave=False): if min_samples_name == "same": min_samples = min_cluster_size else: min_samples = min_samples_name clusterer = hdbscan.HDBSCAN(min_samples=min_samples, min_cluster_size=min_cluster_size, approx_min_span_tree=False) labels_list = [] for batch_id in tqdm(range(data.shape[0]), desc="Processing batch", leave=False): clusterer.fit(data[batch_id].reshape(-1, data.shape[-1])) labels_list.append( self.assign_noise( data[batch_id].reshape(-1, data.shape[-1]), clusterer.labels_).reshape(data.shape[1:-1])) tqdm.write( "Done with clustering batch {}".format(batch_id)) labels = np.array(labels_list) self.save_labels(labels, min_cluster_size, min_samples) # compute metrics results = { "parameters": { "min_samples": min_samples, "min_cluster_size": min_cluster_size }, "scores": compute_metrics(labels, gt.copy()) } # save results np.save( os.path.join( self.target_dir, "hdbscan_scores_min_samples_{}_min_cluster_size_{}". format(min_cluster_size, min_samples)), np.array([ results["scores"]["CREMI_score"], results["scores"]["arand"], results["scores"]["voi"][0], results["scores"]["voi"][1] ])) results_list.append(results) tqdm.write( "Done with min_samples {} and min_cluster_size {}".format( min_samples, min_cluster_size)) return sorted(results_list, key=lambda x: x["scores"]["CREMI_score"])[0]
def run(self, data, gt): results_list = [] # for each batch compute complete linkage tree once, cut it at various thresholds labels_list = [] for batch_id in tqdm(range(data.shape[0]), desc="Processing batch", leave=False): labels_batch = [] complete_linkage = complete(data[batch_id].reshape( -1, data.shape[-1])) for threshold in tqdm(self.thresholds, desc="Cutting at thresholds", leave=False): labels_threshold_slice = fcluster(complete_linkage, t=threshold, criterion="distance") labels_threshold_slice = labels_threshold_slice.reshape( *data.shape[1:-1]) labels_batch.append(labels_threshold_slice) labels_list.append(labels_batch) tqdm.write("Done with clustering batch {}".format(batch_id)) labels = np.array(labels_list) # compute scores, save labels and scores for each threshold for i, threshold in enumerate(self.thresholds): np.save( os.path.join( self.target_dir, "complete_linkage_labels_threshold_{}.npy".format( threshold)), labels[:, i, ...]) # compute metrics results = { "parameters": { "threshold": threshold }, "scores": compute_metrics(labels[:, i, ...], gt.copy()) } # save results np.save( os.path.join( self.target_dir, "complete_linkage_scores_threshold_{}".format(threshold)), np.array([ results["scores"]["CREMI_score"], results["scores"]["arand"], results["scores"]["voi"][0], results["scores"]["voi"][1] ])) results_list.append(results) return sorted(results_list, key=lambda x: x["scores"]["CREMI_score"])[0]
def main(args): labels: pd.DataFrame = pd.read_pickle(args.labels) if "narration_id" in labels.columns: labels.set_index("narration_id", inplace=True) labels = add_action_class_column(labels) unseen_participants: np.ndarray = pd.read_csv( args.unseen_participant_ids_csv, index_col="participant_id" ).index.values tail_verb_classes: np.ndarray = pd.read_csv( args.tail_verb_classes_csv, index_col="verb" ).index.values tail_noun_classes: np.ndarray = pd.read_csv( args.tail_noun_classes_csv, index_col="noun" ).index.values results = load_results(args.results) verb_output = results["verb_output"] noun_output = results["noun_output"] narration_ids = results["narration_id"] scores = { "verb": verb_output, "noun": noun_output, } (verbs, nouns), _scores = compute_action_scores( scores["verb"], scores["noun"], top_n=100 ) scores["action"] = [ { action_id_from_verb_noun(verb, noun): score for verb, noun, score in zip(segment_verbs, segment_nouns, segment_score) } for segment_verbs, segment_nouns, segment_score in zip(verbs, nouns, _scores) ] accuracies = compute_metrics( labels.loc[narration_ids], scores, tail_verb_classes, tail_noun_classes, unseen_participants, ) display_metrics = dict() for split in accuracies.keys(): for task in ["verb", "noun", "action"]: task_accuracies = accuracies[split][task] for k, task_accuracy in zip((1, 5), task_accuracies): display_metrics[f"{split}_{task}_accuracy_at_{k}"] = task_accuracy display_metrics = { metric: float(value * 100) for metric, value in display_metrics.items() } print(yaml.dump(display_metrics))
def computeAverageMetrics(imfeats, recipefeats, k, t, forceorder=False): """Computes retrieval metrics for two sets of features Parameters ---------- imfeats : np.ndarray [n x d] The image features.. recipefeats : np.ndarray [n x d] The recipe features. k : int Ranking size. t : int Number of evaluations to run (function returns the average). forceorder : bool Whether to force a particular order instead of picking random samples Returns ------- dict Dictionary with metric values for all t runs. """ glob_metrics = {} i = 0 for _ in range(t): if forceorder: # pick the same samples in the same order for evaluation # forceorder is only True when the function is used during training sub_ids = np.array(range(i, i + k)) i += k else: sub_ids = random.sample(range(0, len(imfeats)), k) imfeats_sub = imfeats[sub_ids, :] recipefeats_sub = recipefeats[sub_ids, :] metrics = compute_metrics(imfeats_sub, recipefeats_sub, recall_klist=(1, 5, 10)) for metric_name, metric_value in metrics.items(): if metric_name not in glob_metrics: glob_metrics[metric_name] = [] glob_metrics[metric_name].append(metric_value) return glob_metrics
def test(self, which_dataset, condition_target): """Test segmentation.""" if which_dataset == 'source': loader = self.source_loader else: loader = self.target_loader # Load the trained generator. self.restore_model(self.G, 'G', self.log_dir) self.restore_model(self.S, 'S', self.log_dir) # Load the trained generator. self.G.eval() self.S.eval() # Evaluate segmentation metrics = {'loss_segm': 0, 'iou': 0, 'accuracy': 0} cm = torch.zeros(2, 2).float().cuda() with torch.no_grad(): for i, (x, gt) in enumerate(loader): # Prepare input images and target masks. x = x.to(self.device) gt = gt.to(self.device) # Segment images condition = 1. if condition_target == 'source' else 0. _, h = self.G( x, condition * torch.ones(x.size(0), 1).to(self.device)) s = self.S(h) metrics['loss_segm'] += self.segm_criterion(s, gt).item() # Update metrics cm = update_cm(cm, s, gt) metrics['loss_segm'] /= len(loader) # Compute metrics metrics = compute_metrics(cm, metrics) print_metrics('TEST ' + which_dataset + ': ', metrics)
def val(self, epoch, model_path, val_loader, val_log, cfg): detecter = Detector(model_path, cfg) mean_precision = 0 mean_recall = 0 sample_count = val_loader.num_samples for i in range(sample_count): image_path, gt_bboxes = val_loader.getitem(i) results = detecter.run(image_path) pre_bboxes = results[1] if len(gt_bboxes) > 0: precision, recall = compute_metrics(pre_bboxes, gt_bboxes) mean_precision += precision mean_recall += recall log_str = "{},{:.6f},{:.6f}\n".format(epoch, mean_precision / sample_count, mean_recall / sample_count) val_log.write(log_str) val_log.flush()
def validation(self, epoch): """Translate images using StarGAN trained on a single dataset.""" # Load the trained generator. self.G.eval() self.D.eval() if self.Df is not None: self.Df.eval() self.S.eval() mix_iter = iter(self.mix_loader_val) source_iter = iter(self.source_loader_val) target_iter = iter(self.target_loader_val) # Evaluate segmentation metrics = { 'S/loss_segm': 0, 'iou': [], 'accuracy': [], 'G/loss': 0, 'G/loss_fake': 0, 'G/loss_cycle': 0, 'G/loss_cls': 0, 'G/loss_id': 0, 'Ge/loss_fdom': 0, 'Ge/loss_frf': 0, 'Ge/loss_ffeat': 0 } cm = torch.zeros(2, 2).float().cuda() i = 0 with torch.no_grad(): while True: # =================================================================================== # # 1. Preprocessing # # =================================================================================== # # Fetch real images and labels. try: x_real, c_org, gt_real = next(mix_iter) except: print( "mix_iter shouldn't have raised this exception in validation" ) # Fetch source images and masks try: x, gt = next(source_iter) if x.size(0) < self.batch_size: raise Exception x_source, gt_source = x, gt except: break # Fetch target images and masks try: x, gt = next(target_iter) if x.size(0) < self.batch_size: raise Exception x_target, gt_target = x, gt except: break x_real = x_real.to(self.device) # Input images. x_source = x_source.to(self.device) x_target = x_target.to(self.device) gt_source = gt_source.to(self.device) gt_target = gt_target.to(self.device) gt_real = gt_real.to(self.device) c_org = c_org.to(self.device) # Original domain labels. # =================================================================================== # # 4. Generator # # =================================================================================== # _, s_source, loss_log = self.G_losses(x_real, c_org, gt_real, x_source, gt_source, x_target, gt_target) cm = update_cm(cm, s_source, gt_source) # =================================================================================== # # 5. Miscellaneous # # =================================================================================== # for k in loss_log: metrics[k] += loss_log[k] i += 1 metrics = compute_metrics(cm, metrics) metrics['G/loss_es'] = metrics['G/loss'] - self.lambda_g * metrics[ 'G/loss_fake'] - self.lambda_fdom * metrics[ 'Ge/loss_fdom'] - self.lambda_frf * metrics['Ge/loss_frf'] pattern = re.compile("(?!iou|accuracy).*") metrics.update( {k: v / i for k, v in metrics.items() if pattern.match(k)}) # Log metrics self.logger.scalar_summary(mode='val', epoch=epoch, **metrics) # Log visualization x_target = x_target.to(self.device) self.tb_images(x_target, torch.zeros(x_target.size(0), 1).to(self.device), epoch, 'val') return metrics['G/loss_es']
def main(args): # Create model directory & other aux folders for logging where_to_save = os.path.join(args.save_dir, args.project_name, args.model_name) checkpoints_dir = os.path.join(where_to_save, 'checkpoints') logs_dir = os.path.join(where_to_save, 'logs') tb_logs = os.path.join(args.save_dir, args.project_name, 'tb_logs', args.model_name) make_dir(where_to_save) make_dir(logs_dir) make_dir(checkpoints_dir) make_dir(tb_logs) if args.tensorboard: logger = Visualizer(tb_logs, name='visual_results') # check if we want to resume from last checkpoint of current model if args.resume: args = pickle.load( open(os.path.join(checkpoints_dir, 'args.pkl'), 'rb')) args.resume = True # logs to disk if not args.log_term: print("Training logs will be saved to:", os.path.join(logs_dir, 'train.log')) sys.stdout = open(os.path.join(logs_dir, 'train.log'), 'w') sys.stderr = open(os.path.join(logs_dir, 'train.err'), 'w') print(args) pickle.dump(args, open(os.path.join(checkpoints_dir, 'args.pkl'), 'wb')) # patience init curr_pat = 0 # Build data loader data_loaders = {} datasets = {} data_dir = args.recipe1m_dir for split in ['train', 'val']: transforms_list = [transforms.Resize((args.image_size))] if split == 'train': # Image preprocessing, normalization for the pretrained resnet transforms_list.append(transforms.RandomHorizontalFlip()) transforms_list.append( transforms.RandomAffine(degrees=10, translate=(0.1, 0.1))) transforms_list.append(transforms.RandomCrop(args.crop_size)) else: transforms_list.append(transforms.CenterCrop(args.crop_size)) transforms_list.append(transforms.ToTensor()) transforms_list.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) transform = transforms.Compose(transforms_list) max_num_samples = max(args.max_eval, args.batch_size) if split == 'val' else -1 data_loaders[split], datasets[split] = get_loader( data_dir, args.aux_data_dir, split, args.maxseqlen, args.maxnuminstrs, args.maxnumlabels, args.maxnumims, transform, args.batch_size, shuffle=split == 'train', num_workers=args.num_workers, drop_last=True, max_num_samples=max_num_samples, use_lmdb=args.use_lmdb, suff=args.suff) ingr_vocab_size = datasets[split].get_ingrs_vocab_size() instrs_vocab_size = datasets[split].get_instrs_vocab_size() # Build the model model = get_model(args, ingr_vocab_size, instrs_vocab_size) keep_cnn_gradients = False decay_factor = 1.0 # add model parameters if args.ingrs_only: params = list(model.ingredient_decoder.parameters()) + list( model.ingredient_encoder.parameters()) elif args.recipe_only: params = list(model.recipe_decoder.parameters()) + list( model.ingredient_encoder.parameters()) else: params = list(model.recipe_decoder.parameters()) + list(model.ingredient_decoder.parameters()) \ + list(model.ingredient_encoder.parameters()) # only train the linear layer in the encoder if we are not transfering from another model if args.transfer_from == '': params += list(model.image_encoder.linear.parameters()) params_cnn = list(model.image_encoder.resnet.parameters()) print("CNN params:", sum(p.numel() for p in params_cnn if p.requires_grad)) print("decoder params:", sum(p.numel() for p in params if p.requires_grad)) # start optimizing cnn from the beginning if params_cnn is not None and args.finetune_after == 0: optimizer = torch.optim.Adam( [{ 'params': params }, { 'params': params_cnn, 'lr': args.learning_rate * args.scale_learning_rate_cnn }], lr=args.learning_rate, weight_decay=args.weight_decay) keep_cnn_gradients = True print("Fine tuning resnet") else: optimizer = torch.optim.Adam(params, lr=args.learning_rate) if args.resume: model_path = os.path.join(args.save_dir, args.project_name, args.model_name, 'checkpoints', 'model.ckpt') optim_path = os.path.join(args.save_dir, args.project_name, args.model_name, 'checkpoints', 'optim.ckpt') optimizer.load_state_dict(torch.load(optim_path, map_location=map_loc)) for state in optimizer.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): state[k] = v.to(device) model.load_state_dict(torch.load(model_path, map_location=map_loc)) if args.transfer_from != '': # loads CNN encoder from transfer_from model model_path = os.path.join(args.save_dir, args.project_name, args.transfer_from, 'checkpoints', 'modelbest.ckpt') pretrained_dict = torch.load(model_path, map_location=map_loc) pretrained_dict = { k: v for k, v in pretrained_dict.items() if 'encoder' in k } model.load_state_dict(pretrained_dict, strict=False) args, model = merge_models(args, model, ingr_vocab_size, instrs_vocab_size) if device != 'cpu' and torch.cuda.device_count() > 1: model = nn.DataParallel(model) model = model.to(device) cudnn.benchmark = True if not hasattr(args, 'current_epoch'): args.current_epoch = 0 es_best = 10000 if args.es_metric == 'loss' else 0 # Train the model start = args.current_epoch for epoch in range(start, args.num_epochs): # save current epoch for resuming if args.tensorboard: logger.reset() args.current_epoch = epoch # increase / decrase values for moving params if args.decay_lr: frac = epoch // args.lr_decay_every decay_factor = args.lr_decay_rate**frac new_lr = args.learning_rate * decay_factor print('Epoch %d. lr: %.5f' % (epoch, new_lr)) set_lr(optimizer, decay_factor) if args.finetune_after != -1 and args.finetune_after < epoch \ and not keep_cnn_gradients and params_cnn is not None: print("Starting to fine tune CNN") # start with learning rates as they were (if decayed during training) optimizer = torch.optim.Adam([{ 'params': params }, { 'params': params_cnn, 'lr': decay_factor * args.learning_rate * args.scale_learning_rate_cnn }], lr=decay_factor * args.learning_rate) keep_cnn_gradients = True for split in ['train', 'val']: if split == 'train': model.train() else: model.eval() total_step = len(data_loaders[split]) loader = iter(data_loaders[split]) total_loss_dict = { 'recipe_loss': [], 'ingr_loss': [], 'eos_loss': [], 'loss': [], 'iou': [], 'perplexity': [], 'iou_sample': [], 'f1': [], 'card_penalty': [] } error_types = { 'tp_i': 0, 'fp_i': 0, 'fn_i': 0, 'tn_i': 0, 'tp_all': 0, 'fp_all': 0, 'fn_all': 0 } torch.cuda.synchronize() start = time.time() for i in range(total_step): img_inputs, captions, ingr_gt, img_ids, paths = loader.next() ingr_gt = ingr_gt.to(device) img_inputs = img_inputs.to(device) captions = captions.to(device) true_caps_batch = captions.clone()[:, 1:].contiguous() loss_dict = {} if split == 'val': with torch.no_grad(): losses = model(img_inputs, captions, ingr_gt) if not args.recipe_only: outputs = model(img_inputs, captions, ingr_gt, sample=True) ingr_ids_greedy = outputs['ingr_ids'] mask = mask_from_eos(ingr_ids_greedy, eos_value=0, mult_before=False) ingr_ids_greedy[mask == 0] = ingr_vocab_size - 1 pred_one_hot = label2onehot( ingr_ids_greedy, ingr_vocab_size - 1) target_one_hot = label2onehot( ingr_gt, ingr_vocab_size - 1) iou_sample = softIoU(pred_one_hot, target_one_hot) iou_sample = iou_sample.sum() / ( torch.nonzero(iou_sample.data).size(0) + 1e-6) loss_dict['iou_sample'] = iou_sample.item() update_error_types(error_types, pred_one_hot, target_one_hot) del outputs, pred_one_hot, target_one_hot, iou_sample else: losses = model(img_inputs, captions, ingr_gt, keep_cnn_gradients=keep_cnn_gradients) if not args.ingrs_only: recipe_loss = losses['recipe_loss'] recipe_loss = recipe_loss.view(true_caps_batch.size()) non_pad_mask = true_caps_batch.ne(instrs_vocab_size - 1).float() recipe_loss = torch.sum(recipe_loss * non_pad_mask, dim=-1) / torch.sum(non_pad_mask, dim=-1) perplexity = torch.exp(recipe_loss) recipe_loss = recipe_loss.mean() perplexity = perplexity.mean() loss_dict['recipe_loss'] = recipe_loss.item() loss_dict['perplexity'] = perplexity.item() else: recipe_loss = 0 if not args.recipe_only: ingr_loss = losses['ingr_loss'] ingr_loss = ingr_loss.mean() loss_dict['ingr_loss'] = ingr_loss.item() eos_loss = losses['eos_loss'] eos_loss = eos_loss.mean() loss_dict['eos_loss'] = eos_loss.item() iou_seq = losses['iou'] iou_seq = iou_seq.mean() loss_dict['iou'] = iou_seq.item() card_penalty = losses['card_penalty'].mean() loss_dict['card_penalty'] = card_penalty.item() else: ingr_loss, eos_loss, card_penalty = 0, 0, 0 loss = args.loss_weight[0] * recipe_loss + args.loss_weight[1] * ingr_loss \ + args.loss_weight[2]*eos_loss + args.loss_weight[3]*card_penalty loss_dict['loss'] = loss.item() for key in loss_dict.keys(): total_loss_dict[key].append(loss_dict[key]) if split == 'train': model.zero_grad() loss.backward() optimizer.step() # Print log info if args.log_step != -1 and i % args.log_step == 0: elapsed_time = time.time() - start lossesstr = "" for k in total_loss_dict.keys(): if len(total_loss_dict[k]) == 0: continue this_one = "%s: %.4f" % ( k, np.mean(total_loss_dict[k][-args.log_step:])) lossesstr += this_one + ', ' # this only displays nll loss on captions, the rest of losses will be in tensorboard logs strtoprint = 'Split: %s, Epoch [%d/%d], Step [%d/%d], Losses: %sTime: %.4f' % ( split, epoch, args.num_epochs, i, total_step, lossesstr, elapsed_time) print(strtoprint) if args.tensorboard: # logger.histo_summary(model=model, step=total_step * epoch + i) logger.scalar_summary( mode=split + '_iter', epoch=total_step * epoch + i, **{ k: np.mean(v[-args.log_step:]) for k, v in total_loss_dict.items() if v }) torch.cuda.synchronize() start = time.time() del loss, losses, captions, img_inputs if split == 'val' and not args.recipe_only: ret_metrics = { 'accuracy': [], 'f1': [], 'jaccard': [], 'f1_ingredients': [], 'dice': [] } compute_metrics( ret_metrics, error_types, ['accuracy', 'f1', 'jaccard', 'f1_ingredients', 'dice'], eps=1e-10, weights=None) total_loss_dict['f1'] = ret_metrics['f1'] if args.tensorboard: # 1. Log scalar values (scalar summary) logger.scalar_summary( mode=split, epoch=epoch, **{k: np.mean(v) for k, v in total_loss_dict.items() if v}) # Save the model's best checkpoint if performance was improved es_value = np.mean(total_loss_dict[args.es_metric]) # save current model as well save_model(model, optimizer, checkpoints_dir, suff='') if (args.es_metric == 'loss' and es_value < es_best) or (args.es_metric == 'iou_sample' and es_value > es_best): es_best = es_value save_model(model, optimizer, checkpoints_dir, suff='best') pickle.dump(args, open(os.path.join(checkpoints_dir, 'args.pkl'), 'wb')) curr_pat = 0 print('Saved checkpoint.') else: curr_pat += 1 if curr_pat > args.patience: break if args.tensorboard: logger.close()
def train(args): # Create directories if not exist. model_dir = os.path.join(args.log_dir, args.exp_name) if not os.path.exists(args.log_dir): os.makedirs(args.log_dir) if not os.path.exists(model_dir): os.makedirs(model_dir) # logs to disk if not args.log_term: print("Training logs will be saved to:", os.path.join(model_dir, 'train.log')) sys.stdout = open(os.path.join(model_dir, 'train.log'), 'w') sys.stderr = open(os.path.join(model_dir, 'train.err'), 'w') # save args pickle.dump(args, open(os.path.join(model_dir, 'args.pkl'), 'wb')) curr_pat = 0 # Data loader source_train = get_loader(args.batch_size, args.source, 'train', args.num_workers, mnistpath=args.mnist_dir, mnistmpath=args.mnist_m_dir, mnistthinpath=args.mnist_thin_dir, source=args.source, colors=False) source_val = get_loader(args.batch_size, args.source, 'val', args.num_workers, mnistpath=args.mnist_dir, mnistmpath=args.mnist_m_dir, mnistthinpath=args.mnist_thin_dir, source=args.source, colors=False) source_test = get_loader(args.batch_size, args.source, 'test', args.num_workers, mnistpath=args.mnist_dir, mnistmpath=args.mnist_m_dir, mnistthinpath=args.mnist_thin_dir, source=args.source, colors=False) target_test = get_loader(args.batch_size, args.target, 'test', args.num_workers, mnistpath=args.mnist_dir, mnistmpath=args.mnist_m_dir, mnistthinpath=args.mnist_thin_dir, source=args.target, colors=False) # Training criterion if args.criterion == 'softiou': criterion = softIoULoss() elif args.criterion == 'crossentropy': criterion = nn.CrossEntropyLoss() else: raise ValueError('Unknown loss') # Build model model = Segmenter(conv_dim=args.conv_dim, repeat_num=args.repeat_num, num_down=args.num_down, bias=True, n_classes=2, drop=args.drop) # optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) # gpus model = model.cuda() cudnn.benchmark = True # Visualizer if args.use_tensorboard: visualizer = Visualizer(model_dir, name='visual_results') # Train the model for epoch in range(0, args.num_epochs): # reset visualizer visualizer.reset() # increase / decrase values for moving params set_lr(optimizer, args.lr_decay) # split loop for split in ['train', 'val']: if split == 'train': loader = source_train model.train() else: loader = source_val model.eval() metrics = {'loss': 0, 'iou': [], 'accuracy': []} cm = torch.from_numpy(np.zeros((2, 2))).float().cuda() total_step = len(loader) torch.cuda.synchronize() start = time.time() # minibatch loop for i, (images, gts) in enumerate(loader): global_iter = total_step * epoch + i # send to cuda images = images.cuda() gts = gts.cuda() loss_dict = {} if split == 'val': with torch.no_grad(): outputs = model(images) else: outputs = model(images) # loss computation loss = criterion(outputs, gts) # update confusion matrix cm = update_cm(cm, outputs, gts) # update dicts loss_dict['loss'] = loss.data metrics['loss'] += loss_dict['loss'] if split == 'train': optimizer.zero_grad() loss.backward() optimizer.step() visualizer.scalar_summary(mode=split, epoch=global_iter, **loss_dict) # end of epoch metrics['loss'] /= total_step str_endepoch = 'total epoch %d; split: %s; loss: %.4f; time: %s' % ( epoch, split, metrics['loss'], time.time() - start) print(str_endepoch) torch.cuda.synchronize() start = time.time() # compute metrics and visualize them metrics = compute_metrics(cm, metrics) if split == 'train': visualizer.scalar_summary(mode=split, epoch=epoch, **{k: v for k, v in metrics.items() if v and k != 'loss'}) if split == 'val': visualizer.scalar_summary(mode=split, epoch=epoch, **metrics) # Save the model checkpoints if performance was improved if epoch == 0 or metrics['loss'] < es_best: es_best = metrics['loss'] torch.save(model.state_dict(), os.path.join( model_dir, 'model.ckpt')) torch.save(optimizer.state_dict(), os.path.join( model_dir, 'optim.ckpt')) curr_pat = 0 else: curr_pat += 1 if curr_pat > args.patience: break visualizer.close() # restore model model = restore_model(model, model_dir) # test on source domain model.eval() cm = torch.from_numpy(np.zeros((2, 2))).float().cuda() metrics = {'loss': 0, 'iou': [], 'accuracy': []} for i, (images, gts) in enumerate(source_test): # send to cuda images = images.cuda() gts = gts.cuda() with torch.no_grad(): outputs = model(images) # loss computation loss = criterion(outputs, gts) # update confusion matrix cm = update_cm(cm, outputs, gts) # update dicts metrics['loss'] += loss.data # compute metrics and visualize them metrics['loss'] /= len(source_test) metrics = compute_metrics(cm, metrics) print_metrics('TEST SOURCE: ', metrics) # test on target domain model.eval() cm = torch.from_numpy(np.zeros((2, 2))).float().cuda() metrics = {'loss': 0, 'iou': [], 'accuracy': []} for i, (images, gts) in enumerate(target_test): # send to cuda images = images.cuda() gts = gts.cuda() with torch.no_grad(): outputs = model(images) # loss computation loss = criterion(outputs, gts) # update confusion matrix cm = update_cm(cm, outputs, gts) # update dicts metrics['loss'] += loss.data # compute metrics and visualize them metrics['loss'] /= len(target_test) metrics = compute_metrics(cm, metrics) print_metrics('TEST TARGET: ', metrics)
def validate_model(Net, seed, mini_batch_size=100, optimizer=optim.Adam, criterion=nn.CrossEntropyLoss(), n_epochs=40, eta=1e-3, lambda_l2=0, alpha=0.5, beta=0.5, plot=True, rotate=False, translate=False, swap_channel=False, GPU=False): """ General : - Train a network model which weights has been initialized with a specific seed over n_epochs - Data is created with the same seed : train,validation and test calling the prologue - Record the train and validation accuracy and loss and can display they evolution curve Input : - Net : A network dictionnary from the <Nets> class - seed : seed for pseudo random number generator used in weight initialization and data loading -> mini_batch_size,optimizer, criterion, n_epochs, eta, lambda_2, alpha, beta see training.py - plot : if true plot the learning curve evolution over the epochs -> default true -> rotate,translate and swap_channels -> data augmentation see loader.py Output : printed loss and accuracy of the network after training on the test set and learning curve if plot true """ # set the pytorch seed torch.manual_seed(seed) torch.cuda.manual_seed(seed) # set the seed for random spliting of the dataset in training and validation random.seed(0) # create the dataset data = PairSetMNIST() train_data = Training_set(data) test_data = Test_set(data) train_data_split = Training_set_split(train_data, rotate, translate, swap_channel) validation_data = Validation_set(train_data) # construct the net type with default parameter if (Net['net_type'] == 'Net2c'): model = Net['net'](nb_hidden=Net['hidden_layers'], dropout_prob=Net['drop_prob']) if (Net['net_type'] == 'LeNet_sharing'): model = Net['net'](nb_hidden=Net['hidden_layers'], dropout_ws=Net['drop_prob_ws'], dropout_comp=Net['drop_prob_comp']) if (Net['net_type'] == 'LeNet_sharing_aux'): # check if any data augmentation has been called # if none construct with tuned parameters without data augmentation # if yes construct with tuned parameters with data augmentation if (rotate == False and translate == False and swap_channel == False): model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'], nbhidden_comp=Net['hidden_layers_comp'], drop_prob_aux=Net['drop_prob_aux'], drop_prob_comp=Net['drop_prob_comp']) else: Net['learning rate'] = Net['learning rate augm'] model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'], nbhidden_comp=Net['hidden_layers_comp'], drop_prob_aux=Net['drop_prob_aux_augm'], drop_prob_comp=Net['drop_prob_comp_augm']) if (Net['net_type'] == 'Google_Net'): model = Net['net'](channels_1x1=Net['channels_1x1'], channels_3x3=Net['channels_3x3'], channels_5x5=Net['channels_5x5'], pool_channels=Net['pool_channels'], nhidden=Net['hidden_layers'], drop_prob_comp=Net['drop_prob_comp'], drop_prob_aux=Net['drop_prob_aux']) if GPU and cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') model = model.to(device) # train the model on the train set and validate at each epoch train_losses, train_acc, valid_losses, valid_acc = train_model( model, train_data_split, validation_data, device, mini_batch_size, optimizer, criterion, n_epochs, Net['learning rate'], lambda_l2, alpha, beta) if plot: learning_curve(train_losses, train_acc, valid_losses, valid_acc) # loss and accuracy of the network on the test test_loss, test_accuracy = compute_metrics(model, test_data, device) print('\nTest Set | Loss: {:.4f} | Accuracy: {:.2f}%\n'.format( test_loss, test_accuracy))
def run(self, data, gt): results_list = [] for bandwidth in tqdm(self.bandwidths, desc="Processing bandwidth", leave=False): MeanShifter = MeanShift(n_iter=self.n_iter, bandwidth=bandwidth, kernel=self.kernel, blurring=self.blurring, use_keops=self.keops) convergence_points = MeanShifter( torch.tensor( data.reshape(data.shape[0], -1, data.shape[-1]))).detach().cpu().numpy() convergence_points.reshape(*data.shape) np.save( os.path.join( self.target_dir, "mean_shift_conv_points_bandwidth_{}.npy".format( bandwidth)), convergence_points) tqdm.write("Obtaining hard clustering") labels = obtain_hard_clusters(convergence_points, [ threshold if not (threshold == "same") else bandwidth for threshold in self.thresholds ]) # compute scores, save labels and scores for each threshold for i, threshold in enumerate(self.thresholds): np.save( os.path.join( self.target_dir, "mean_shift_labels_bandwidth_{}_threshold_{}.npy". format(bandwidth, threshold)), labels[:, i, ...]) # compute metrics results = { "parameters": { "bandwidth": bandwidth, "threshold": threshold }, "scores": compute_metrics(labels[:, i, ...], gt.copy()) } # save results np.save( os.path.join( self.target_dir, "mean_shift_scores_bandwidth_{}_threshold_{}".format( bandwidth, threshold)), np.array([ results["scores"]["CREMI_score"], results["scores"]["arand"], results["scores"]["voi"][0], results["scores"]["voi"][1] ])) results_list.append(results) tqdm.write("Done with bandwidth {}".format(bandwidth)) return sorted(results_list, key=lambda x: x["scores"]["CREMI_score"])[0]
def main(): params = parseyaml() if params['arch'] == 'Generator': device = to_gpu(ngpu=params['n_gpu']) if params['image_size'] == 64: netG = Generator(ngpu=0, nz=256, ngf=64, nc=64).to(device) elif params['image_size'] == 128: netG = Generator_128(ngpu=0, nz=256, ngf=64, nc=64).to(device) elif params['image_size'] == 256: netG = Generator_256(ngpu=0, nz=256, ngf=64, nc=64).to(device) netG.apply(weights_init) netG.load_state_dict(torch.load(params['path'])) for i in range(params['quantity']): fixed_noise = torch.randn(64, 256, 1, 1, device=device) fakes = netG(fixed_noise) for j in range(len(fakes)): save_image(fakes[j], params['out'] + params['run'] + '_' + str(i) + '_' + str(j) + '_img.png') else: dataloader = dataLoader( path=params['path'], image_size=params['image_size'], batch_size=params['batch_size'], workers=params['loader_workers']) device = to_gpu(ngpu=params['n_gpu']) if params['arch'] == 'DCGAN': if params['image_size'] == 64: netG = Generator(ngpu=params['n_gpu'], nz=params['latent_vector'], ngf=params['gen_feature_maps'], nc=params['number_channels']).to(device) netD = Discriminator(params['n_gpu'], nc=params['number_channels'], ndf=params['dis_feature_maps']).to(device) elif params['image_size'] == 128: netG = Generator_128(ngpu=params['n_gpu'], nz=params['latent_vector'], ngf=params['gen_feature_maps'], nc=params['number_channels']).to(device) netD = Discriminator_128(params['n_gpu'], nc=params['number_channels'], ndf=params['dis_feature_maps']).to(device) elif params['image_size'] == 256: netG = Generator_256(ngpu=params['n_gpu'], nz=params['latent_vector'], ngf=params['gen_feature_maps'], nc=params['number_channels']).to(device) netD = Discriminator_256(params['n_gpu'], nc=params['number_channels'], ndf=params['dis_feature_maps']).to(device) elif params['arch'] == 'SNGAN': if params['image_size'] == 64: netG = Generator(ngpu=params['n_gpu'], nz=params['latent_vector'], ngf=params['gen_feature_maps'], nc=params['number_channels']).to(device) netD = Discriminator_SN(params['n_gpu'], nc=params['number_channels'], ndf=params['dis_feature_maps']).to(device) elif params['image_size'] == 128: netG = Generator_128(ngpu=params['n_gpu'], nz=params['latent_vector'], ngf=params['gen_feature_maps'], nc=params['number_channels']).to(device) netD = Discriminator_SN_128(params['n_gpu'], nc=params['number_channels'], ndf=params['dis_feature_maps']).to(device) elif params['image_size'] == 256: netG = Generator_256(ngpu=params['n_gpu'], nz=params['latent_vector'], ngf=params['gen_feature_maps'], nc=params['number_channels']).to(device) netD = Discriminator_SN_256(params['n_gpu'], nc=params['number_channels'], ndf=params['dis_feature_maps']).to(device) if (device.type == 'cuda') and (params['n_gpu'] > 1): netG = nn.DataParallel(netG, list(range(params['n_gpu']))) if (device.type == 'cuda') and (params['n_gpu'] > 1): netD = nn.DataParallel(netD, list(range(params['n_gpu']))) netG.apply(weights_init) netD.apply(weights_init) print(netG) print(netD) criterion = nn.BCELoss() fixed_noise = torch.randn(params['image_size'], params['latent_vector'], 1, 1, device=device) if params['learning_rate'] >= 1: optimizerD = optim.Adam(netD.parameters(), lr=0.0002 * params['learning_rate'], betas=( params['beta_adam'], 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=0.0002, betas=( params['beta_adam'], 0.999)) else: optimizerD = optim.Adam(netD.parameters(), lr=params['learning_rate'], betas=( params['beta_adam'], 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=params['learning_rate'], betas=( params['beta_adam'], 0.999)) G_losses, D_losses, img_list, img_list_only = training_loop(num_epochs=params['num_epochs'], dataloader=dataloader, netG=netG, netD=netD, device=device, criterion=criterion, nz=params[ 'latent_vector'], optimizerG=optimizerG, optimizerD=optimizerD, fixed_noise=fixed_noise, out=params['out'] + params['run'] + '_') loss_plot(G_losses=G_losses, D_losses=D_losses, out=params['out'] + params['run'] + '_') image_grid(dataloader=dataloader, img_list=img_list, device=device, out=params['out'] + params['run'] + '_') compute_metrics(real=next(iter(dataloader)), fakes=img_list_only, size=params['image_size'], out=params['out'] + params['run'] + '_')
def main(args): where_to_save = os.path.join(args.save_dir, args.project_name, args.model_name) checkpoints_dir = os.path.join(where_to_save, 'checkpoints') logs_dir = os.path.join(where_to_save, 'logs') if not args.log_term: print("Eval logs will be saved to:", os.path.join(logs_dir, 'eval.log')) sys.stdout = open(os.path.join(logs_dir, 'eval.log'), 'w') sys.stderr = open(os.path.join(logs_dir, 'eval.err'), 'w') vars_to_replace = [ 'greedy', 'recipe_only', 'ingrs_only', 'temperature', 'batch_size', 'maxseqlen', 'get_perplexity', 'use_true_ingrs', 'eval_split', 'save_dir', 'aux_data_dir', 'recipe1m_dir', 'project_name', 'use_lmdb', 'beam' ] store_dict = {} for var in vars_to_replace: store_dict[var] = getattr(args, var) #args = pickle.load(open(os.path.join(checkpoints_dir, 'args.pkl'), 'rb')) for var in vars_to_replace: setattr(args, var, store_dict[var]) print(args) transforms_list = [] transforms_list.append(transforms.Resize((args.crop_size))) transforms_list.append(transforms.CenterCrop(args.crop_size)) transforms_list.append(transforms.ToTensor()) transforms_list.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) # Image preprocessing transform = transforms.Compose(transforms_list) # data loader data_dir = args.recipe1m_dir data_loader, dataset = get_loader(data_dir, args.aux_data_dir, args.eval_split, args.maxseqlen, args.maxnuminstrs, args.maxnumlabels, args.maxnumims, transform, args.batch_size, shuffle=False, num_workers=args.num_workers, drop_last=False, max_num_samples=-1, use_lmdb=args.use_lmdb, suff=args.suff) ingr_vocab_size = dataset.get_ingrs_vocab_size() instrs_vocab_size = dataset.get_instrs_vocab_size() args.numgens = 1 # Build the model model = get_model(args, ingr_vocab_size, instrs_vocab_size) model_path = os.path.join(args.save_dir, args.project_name, args.model_name, 'checkpoints', 'modelbest.ckpt') # overwrite flags for inference model.recipe_only = args.recipe_only model.ingrs_only = args.ingrs_only # Load the trained model parameters model.load_state_dict(torch.load(model_path, map_location=map_loc)) model.eval() model = model.to(device) results_dict = {'recipes': {}, 'ingrs': {}, 'ingr_iou': {}} captions = {} iou = [] error_types = { 'tp_i': 0, 'fp_i': 0, 'fn_i': 0, 'tn_i': 0, 'tp_all': 0, 'fp_all': 0, 'fn_all': 0 } perplexity_list = [] n_rep, th = 0, 0.3 for i, (img_inputs, true_caps_batch, ingr_gt, imgid, impath) in tqdm(enumerate(data_loader)): ingr_gt = ingr_gt.to(device) true_caps_batch = true_caps_batch.to(device) true_caps_shift = true_caps_batch.clone()[:, 1:].contiguous() img_inputs = img_inputs.to(device) true_ingrs = ingr_gt if args.use_true_ingrs else None for gens in range(args.numgens): with torch.no_grad(): if args.get_perplexity: losses = model(img_inputs, true_caps_batch, ingr_gt, keep_cnn_gradients=False) recipe_loss = losses['recipe_loss'] recipe_loss = recipe_loss.view(true_caps_shift.size()) non_pad_mask = true_caps_shift.ne(instrs_vocab_size - 1).float() recipe_loss = torch.sum(recipe_loss * non_pad_mask, dim=-1) / torch.sum(non_pad_mask, dim=-1) perplexity = torch.exp(recipe_loss) perplexity = perplexity.detach().cpu().numpy().tolist() perplexity_list.extend(perplexity) else: outputs = model.sample(img_inputs, args.greedy, args.temperature, args.beam, true_ingrs) if not args.recipe_only: fake_ingrs = outputs['ingr_ids'] pred_one_hot = label2onehot(fake_ingrs, ingr_vocab_size - 1) target_one_hot = label2onehot(ingr_gt, ingr_vocab_size - 1) iou_item = torch.mean( softIoU(pred_one_hot, target_one_hot)).item() iou.append(iou_item) update_error_types(error_types, pred_one_hot, target_one_hot) fake_ingrs = fake_ingrs.detach().cpu().numpy() for ingr_idx, fake_ingr in enumerate(fake_ingrs): iou_item = softIoU( pred_one_hot[ingr_idx].unsqueeze(0), target_one_hot[ingr_idx].unsqueeze(0)).item() results_dict['ingrs'][imgid[ingr_idx]] = [] results_dict['ingrs'][imgid[ingr_idx]].append( fake_ingr) results_dict['ingr_iou'][ imgid[ingr_idx]] = iou_item if not args.ingrs_only: sampled_ids_batch = outputs['recipe_ids'] sampled_ids_batch = sampled_ids_batch.cpu().detach( ).numpy() for j, sampled_ids in enumerate(sampled_ids_batch): score = compute_score(sampled_ids) if score < th: n_rep += 1 if imgid[j] not in captions.keys(): results_dict['recipes'][imgid[j]] = [] results_dict['recipes'][imgid[j]].append( sampled_ids) if args.get_perplexity: print(len(perplexity_list)) print(np.mean(perplexity_list)) else: if not args.recipe_only: ret_metrics = { 'accuracy': [], 'f1': [], 'jaccard': [], 'f1_ingredients': [] } compute_metrics(ret_metrics, error_types, ['accuracy', 'f1', 'jaccard', 'f1_ingredients'], eps=1e-10, weights=None) for k, v in ret_metrics.items(): print(k, np.mean(v)) if args.greedy: suff = 'greedy' else: if args.beam != -1: suff = 'beam_' + str(args.beam) else: suff = 'temp_' + str(args.temperature) results_file = os.path.join( args.save_dir, args.project_name, args.model_name, 'checkpoints', args.eval_split + '_' + suff + '_gencaps.pkl') print(results_file) pickle.dump(results_dict, open(results_file, 'wb')) print("Number of samples with excessive repetitions:", n_rep)
def main(args): # Get models to test from models_path models_to_test = glob.glob(args.models_path + '/*.ckpt') # To store results mat_f1 = np.zeros((len(models_to_test), )) mat_f1_c = np.zeros((len(models_to_test), )) mat_f1_i = np.zeros((len(models_to_test), )) if not os.path.exists(args.save_results_path): os.makedirs(args.save_results_path) print('Results will be saved here: ' + args.save_results_path) # Extract model names models_to_test = [m for m in models_to_test if args.dataset in m] model_names = [re.split(r'[/]', m)[-1] for m in models_to_test] # Iterate over models to test for k, m in enumerate(models_to_test): print('---------------------------------------------') print('Evaluating ' + model_names[k]) # Load model checkpoint = torch.load(m, map_location=map_loc) model_args = checkpoint['args'] # Image pre-processing transforms_list = [] transforms_list.append(transforms.Resize(model_args.image_size)) transforms_list.append(transforms.CenterCrop(model_args.crop_size)) transforms_list.append(transforms.ToTensor()) transforms_list.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) transform = transforms.Compose(transforms_list) # Load data datapaths = json.load(open('../configs/datapaths.json')) dataset_root = datapaths[model_args.dataset] data_loader, dataset = get_loader( dataset=model_args.dataset, dataset_root=dataset_root, split=args.eval_split, transform=transform, batch_size=args.batch_size, include_eos=(model_args.decoder != 'ff'), shuffle=False, num_workers=8, drop_last=False, shuffle_labels=False) vocab_size = len(dataset.get_vocab()) print('Vocabulary size is {}'.format(vocab_size)) print('Dataset {} split contains {} images'.format( args.eval_split, len(dataset))) # Load model model = get_model(model_args, vocab_size) model.load_state_dict(checkpoint['state_dict']) # Eval model.eval() model = model.to(device) total_step = len(data_loader) print('Number of iterations is {}'.format(total_step)) overall_error_counts = { 'tp_c': 0, 'fp_c': 0, 'fn_c': 0, 'tn_c': 0, 'tp_all': 0, 'fp_all': 0, 'fn_all': 0 } error_counts_per_card = {} f1s_image_per_card = {} card_l1_err = [] f1s_image = [] card_accs = [] for l, (img_inputs, target) in tqdm(enumerate(data_loader)): img_inputs = img_inputs.to(device) with torch.no_grad(): # get model predictions # predictions format can either be a matrix of size batch_size x maxnumlabels, where # each row contains the integer labels of an image, followed by pad_value # or a list of sublists, where each sublist contains the integer labels of an image # and len(list) = batch_size and len(sublist) is variable _, predictions = model(img_inputs, maxnumlabels=model_args.maxnumlabels, compute_predictions=True) # convert model predictions and targets to k-hots pred_k_hots = label2_k_hots( predictions, vocab_size - 1, remove_eos=(model_args.decoder != 'ff')) target_k_hots = label2_k_hots( target, vocab_size - 1, remove_eos=(model_args.decoder != 'ff')) # update overall and per class error counts update_error_counts(overall_error_counts, pred_k_hots, target_k_hots) # get per-image error counts for i in range(pred_k_hots.size(0)): # compute per image metrics image_error_counts = { 'tp_c': 0, 'fp_c': 0, 'fn_c': 0, 'tn_c': 0, 'tp_all': 0, 'fp_all': 0, 'fn_all': 0 } update_error_counts(image_error_counts, pred_k_hots[i].unsqueeze(0), target_k_hots[i].unsqueeze(0)) image_metrics = compute_metrics(image_error_counts, which_metrics=['f1']) f1s_image.append(image_metrics['f1']) # compute overall and per class metrics overall_metrics = compute_metrics(overall_error_counts, ['f1', 'c_f1'], weights=None) overall_metrics['f1_i'] = np.mean(f1s_image) print(overall_metrics) # save results mat_f1[k] = overall_metrics['f1'] mat_f1_c[k] = overall_metrics['c_f1'] mat_f1_i[k] = overall_metrics['f1_i'] print('Saving results...') data = { 'Model': model_names, 'f1': mat_f1, 'f1_c': mat_f1_c, 'f1_i': mat_f1_i } df = pd.DataFrame(data) df.to_csv(os.path.join(args.save_results_path, 'results.csv'))
def train_model(model, train_data, validation_data, device, mini_batch_size=100, optimizer=optim.Adam, criterion=nn.CrossEntropyLoss(), n_epochs=40, eta=1e-3, lambda_l2=0, alpha=0.5, beta=0.5): """ Train a neural network model and record train/validation history Input : - model : Neural network class to train - train_data : data used for training the network : input,target and classes - Validation data : unseen data used to evaluate the performance of the network at each epoch on evaluation mode -> input,target and classes - mini_batch_size : batch size in which the data is splitted by the data loader -> default 100 - optimizer : optimizer used to optimize the network -> default Adam -criterion : loss functionto minimize to train the network -> default cross entropy loss - n_epochs : number of steps to optimize the network -> default 40 - eta : learning rate used by the optimizer -> default 10e-3 - lambda_l2 : weight penalty term (weight decay) -> default 0 - alpha : weight term of the binary loss in the overall loss-> default 0.5 - beta : weight term of the auxiliary loss in the overall loss-> default 0.5 Output : - List of the train accuracy at each epoch - List of the train losses at each epoch - List of the validation accuracy at each epoch - List of the validation loss at each epoch """ # Accuracy and loss history of the train and validation data train_acc = [] train_losses = [] valid_acc = [] valid_losses = [] # optimizer class initialized with the parameters passed in the constructor optimizer = optimizer(model.parameters(), lr=eta, weight_decay=lambda_l2) # data loader train_loader = DataLoader(train_data, batch_size=mini_batch_size, shuffle=True) for e in range(n_epochs): epoch_loss = 0 # set the model to train mode model.train(True) for i, data in enumerate(train_loader, 0): # get the data from the batch input_, target_, classes_ = data input_ = input_.to(device) target_ = target_.to(device) classes_ = classes_.to(device) # check the name of the model to know if the output contain auxiliary loss if (model.__class__.__name__ == 'LeNet_sharing_aux' or model.__class__.__name__ == 'Google_Net'): # get model output class_1, class_2, out = model(input_) # compute the different losses aux_loss1 = criterion(class_1, classes_[:, 0]) aux_loss2 = criterion(class_2, classes_[:, 1]) out_loss = criterion(out, target_) # Overall loss to minimize net_loss = (alpha * (out_loss) + beta * (aux_loss1 + aux_loss2)) else: # get the model output out = model(input_) # Compute the overall loss to minimize net_loss = criterion(out, target_) # overral loss on the batch epoch_loss += net_loss # backward optimizer.zero_grad() net_loss.backward() # gradient step optimizer.step() # compute the loss and accuracy on the whole batch for the training for the epoch tr_loss, tr_acc = compute_metrics(model, train_data, device) # compute the loss and accuracy on the validation set for the epoch val_loss, val_acc = compute_metrics(model, validation_data, device) # Save the metrics in the list train_losses.append(tr_loss) train_acc.append(tr_acc) valid_acc.append(val_acc) valid_losses.append(val_loss) return train_losses, train_acc, valid_losses, valid_acc
def evaluate(args, model, prefix="", save_preds=False): # Loop to handle MNLI double evaluation (matched, mis-matched) eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else ( args.task_name, ) eval_outputs_dirs = (args.output_dir, args.output_dir + "-MM") if args.task_name == "mnli" else ( args.output_dir, ) results = {} for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): eval_dataset, _ = load_after_examples(args, eval_task, args.aux_name, mode="dev") if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: os.makedirs(eval_output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max( 1, args.n_gpu) # Note that DistributedSampler samples randomly eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) eval_loss, dom_loss = 0.0, 0.0 nb_eval_steps = 0 preds = None out_label_ids = None for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3] } if args.model_type != "distilbert": inputs["token_type_ids"] = ( batch[2] if args.model_type in ["bert", "xlnet", "albert"] else None ) # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids outputs = model(**inputs, aux=False) tmp_eval_loss, tmp_dom_loss, logits = outputs[:3] eval_loss += tmp_eval_loss.mean().item() dom_loss += tmp_dom_loss.mean().item() nb_eval_steps += 1 if preds is None: preds = logits.detach().cpu().numpy() out_label_ids = inputs["labels"].detach().cpu().numpy()[:, 0] else: preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) out_label_ids = np.append( out_label_ids, inputs["labels"].detach().cpu().numpy()[:, 0], axis=0) eval_loss = eval_loss / nb_eval_steps if args.eval_domain: _, eval_aux_dataset = load_after_examples(args, eval_task, args.aux_name, mode="dev") if not os.path.exists(eval_output_dir) and args.local_rank in [ -1, 0 ]: os.makedirs(eval_output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max( 1, args.n_gpu) # Note that DistributedSampler samples randomly eval_aux_sampler = SequentialSampler(eval_aux_dataset) eval_aux_dataloader = DataLoader(eval_aux_dataset, sampler=eval_aux_sampler, batch_size=args.eval_batch_size) # multi-gpu eval if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) # Eval! logger.info( "***** Running auxiliary evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_aux_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) for batch in tqdm(eval_aux_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3] } if args.model_type != "distilbert": inputs["token_type_ids"] = ( batch[2] if args.model_type in ["bert", "xlnet", "albert"] else None ) # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids outputs = model(**inputs, aux=True) _, tmp_dom_loss, logits = outputs[:3] dom_loss += tmp_dom_loss.mean().item() nb_eval_steps += 1 dom_loss = dom_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) elif args.output_mode == "regression": preds = np.squeeze(preds) if save_preds: metrics = np.array(preds == out_label_ids, dtype=int) with open(eval_output_dir + '/preds.tsv', 'a+') as out_file: tsv_writer = csv.writer(out_file, delimiter='\t') tsv_writer.writerow(["preds", "labels"]) for pred, label, metric in zip(preds, out_label_ids, metrics): tsv_writer.writerow([pred, label]) result = compute_metrics(eval_task, preds, out_label_ids) # AfterBERT result.update({"loss": eval_loss}) if args.eval_domain: result.update({"dom loss": dom_loss}) results.update(result) output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") with open(output_eval_file, "a+") as writer: logger.info("***** Eval results {} *****".format(prefix)) for key in sorted(result.keys()): writer.write("%s = %s\n" % (key, str(result[key]))) return results
def grid_search_aux(lrs, drop_prob_aux, drop_prob_comp, seeds, mini_batch_size=100, optimizer=optim.Adam, criterion=nn.CrossEntropyLoss(), n_epochs=40, lambda_l2=0, alpha=0.5, beta=0.5, rotate=False, translate=False, swap_channel=False, GPU=False): """ General : Iterate over combinations of parameters list to optimize and repeat a 10 round training/validation procedure at each combination -> Select the combination with the highest validation accuracy for a LeNet_sharing_aux network => only called in the <Nets> class by the Tune_LeNet_sharing_aux function Input : - lrs : list of learning rate - drop_prob_aux : list of dropout rate for the CNN auxiliary part - drop_prob_comp : list of dropout rate for the FC layer part for binary classification - seeds : list of seeds for statistics -> mini_batch_size,optimizer, criterion, n_epochs, eta, lambda_2, alpha, beta see training.py -> rotate,translate and swap_channels -> data augmentation see loader.py Ouput : - train_results : A (len(lrs)xlen(drop_prob_aux)xlen(drop_prob_comp)xlen(seeds),4,n_epochs) tensor len() -> number of parameters or seed 4 -> train loss ,train accuracy, validation loss, validation accuracy n_epochs -> evolution during training - test_losses : A tensor of shape (10,) containing the test loss at each seed - test_accuracies : A tensor of shape (10,) containing the test loss at each seed - opt_lr : tuned value for learning rate - opt_prob_aux : tuned value for drop_prob_aux - opt_prob_comp : tuned value for drop_prob_comp """ # tensor to record the metrics train_results = torch.empty(len(lrs), len(drop_prob_aux), len(drop_prob_comp), len(seeds), 4, n_epochs) test_losses = torch.empty(len(lrs), len(drop_prob_aux), len(drop_prob_comp), len(seeds)) test_accuracies = torch.empty(len(lrs), len(drop_prob_aux), len(drop_prob_comp), len(seeds)) # iterate over the parameter combiantion for each seed in seeds for idz, eta in enumerate(lrs): for idx, prob_aux in enumerate(drop_prob_aux): for idy, prob_comp in enumerate(drop_prob_comp): for n, seed in enumerate(seeds): print( ' lr : {:.4f}, prob aux : {:.2f}, prob comp : {:.2f} (n= {:d})' .format(eta, prob_aux, prob_comp, n)) # set seed torch.manual_seed(seed) torch.cuda.manual_seed(seed) #set the random seed random.seed(0) # create the data data = PairSetMNIST() train_data = Training_set(data) test_data = Test_set(data) train_data_split = Training_set_split( train_data, rotate, translate, swap_channel) validation_data = Validation_set(train_data) # create the network model = LeNet_sharing_aux(drop_prob_aux=prob_aux, drop_prob_comp=prob_comp) if GPU and cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') model = model.to(device) # train the network train_losses, train_acc, valid_losses, valid_acc = train_model( model, train_data_split, validation_data, device, mini_batch_size, optimizer, criterion, n_epochs, eta, lambda_l2, alpha, beta) # store train and test results train_results[idz, idx, idy, n, ] = torch.tensor( [train_losses, train_acc, valid_losses, valid_acc]) test_loss, test_acc = compute_metrics( model, test_data, device) test_losses[idz, idx, idy, n] = test_loss test_accuracies[idz, idx, idy, n] = test_acc # compute the validation mean accuracy and standard deviation of the accuracy validation_grid_mean_acc = torch.mean(train_results[:, :, :, :, 3, 39], dim=3) validation_grid_std_acc = torch.std(train_results[:, :, :, :, 3, 39], dim=3) # compute thetest mean accuracy and standard deviation of the accuracy train_grid_mean_acc = torch.mean(train_results[:, :, :, :, 1, 39], dim=3) train_grid_std_acc = torch.std(train_results[:, :, :, :, 1, 39], dim=3) # get the indices of the parameter with the highest mean validation accuracy idx = torch.where( validation_grid_mean_acc == validation_grid_mean_acc.max()) if len(idx[0]) >= 2: idx = idx[0] # get the tuned parameters opt_lr = lrs[idx[0].item()] opt_prob_aux = drop_prob_aux[idx[1].item()] opt_prob_comp = drop_prob_comp[idx[2].item()] print( 'Best mean validation accuracy on {:d} seeds : {:.2f}%, std = {:.2f} with: learning rate = {:.4f} dropout rate aux = {:.2f} and dropout rate comp = {:.2f}' .format( len(seeds), validation_grid_mean_acc[idx[0].item(), idx[1].item(), idx[2].item()], validation_grid_std_acc[idx[0].item(), idx[1].item(), idx[2].item()], opt_lr, opt_prob_aux, opt_prob_comp)) return train_results, test_losses, test_accuracies, opt_lr, opt_prob_aux, opt_prob_comp ###########################################################################################################################################
def evaluate(args, model, tokenizer, mode="", prefix=""): # Loop to handle MNLI double evaluation (matched, mis-matched) eval_task_names = (("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name, )) eval_outputs_dirs = ((args.output_dir, args.output_dir + "-MM") if args.task_name == "mnli" else (args.output_dir, )) results = {} for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, mode) if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: os.makedirs(eval_output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max( 1, args.n_gpu) # Note that DistributedSampler samples randomly eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) eval_loss = 0.0 nb_eval_steps = 0 preds = None out_label_ids = None for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3] } # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use # segment_ids if args.model_type != "distilbert": inputs["token_type_ids"] = (batch[2] if args.model_type in ["bert", "xlnet", "albert" ] else None) outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] eval_loss += tmp_eval_loss.mean().item() nb_eval_steps += 1 if preds is None: preds = logits.detach().cpu().numpy() out_label_ids = inputs["labels"].detach().cpu().numpy() else: preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) out_label_ids = np.append( out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0) eval_loss = eval_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) elif args.output_mode == "regression": preds = np.squeeze(preds) if args.write_preds: output_path_file = os.path.join(eval_output_dir, prefix, "predictions.txt") logger.info("***** Writing Predictions to " "{} *****".format(output_path_file)) write_predictions(output_path_file, eval_task, preds) result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results {} *****".format(prefix)) for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) return results
def evaluate(args, model, tokenizer, prefix=""): # Loop to handle MNLI double evaluation (matched, mis-matched) eval_task_names = (args.task_name, ) eval_outputs_dirs = (args.output_dir, ) results = [] for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: os.makedirs(eval_output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max( 1, args.n_gpu) # Note that DistributedSampler samples randomly eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval if args.n_gpu > 1: model = torch.nn.DataParallel(model) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) eval_loss = 0.0 nb_eval_steps = 0 preds = None out_label_ids = None processor = processors[args.task_name]() # output file for sequence tagging only data_name = args.data_dir.split('/')[-2] output_file = os.path.join( os.path.join(eval_output_dir, prefix), "sequence_tagging_predictions_" + data_name + ".conll") if os.path.isfile(output_file): os.remove(output_file) logger.info(" Deleted existing evaluation file!") for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3] } if args.model_type != "distilbert": inputs["token_type_ids"] = ( batch[2] if args.model_type in ["bert", "xlnet", "albert"] else None ) # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids outputs = model(**inputs) if args.output_mode == "sequencetagging": tmp_eval_loss, emissions, path = outputs[:3] logits_tensor = path.detach().cpu().numpy() labels_tensor = inputs["labels"].detach().cpu().numpy() # Write output as conll file tokenizer.batch_to_conll(inputs["input_ids"], logits_tensor, labels_tensor, processor, output_file) logits = logits_tensor.flatten() labels = labels_tensor.flatten() else: tmp_eval_loss, logits = outputs[:2] logits = logits.detach().cpu().numpy() labels = inputs["labels"].detach().cpu().numpy() eval_loss += tmp_eval_loss.mean().item() nb_eval_steps += 1 if preds is None: preds = logits out_label_ids = labels else: preds = np.append(preds, logits, axis=0) out_label_ids = np.append(out_label_ids, labels, axis=0) eval_loss = eval_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) elif args.output_mode == "regression": preds = np.squeeze(preds) result = compute_metrics(eval_task, preds, out_label_ids) results.append(result) output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results {} *****".format(prefix)) logger.info(results[0]) writer.write(results[0]) return results
def evaluate(args, model, tokenizer, prefix=""): eval_task = args.task_name results = {} # Change output_dir with hyper parameters. hyper_param_str = get_eval_folder_name(args) eval_output_dir = os.path.join(args.output_dir, hyper_param_str) if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: os.makedirs(eval_output_dir) eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) eval_loss = 0.0 nb_eval_steps = 0 all_guids = None all_probs = None preds = None out_label_ids = None for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) guids = batch[0] with torch.no_grad(): inputs = {"input_ids": batch[1], "attention_mask": batch[2], "labels": batch[4]} if args.model_type != "distilbert" and args.model_type != "bart": inputs["token_type_ids"] = ( batch[3] if args.model_type in ["bert", "xlnet", "albert"] else None ) # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] eval_loss += tmp_eval_loss.mean().item() nb_eval_steps += 1 if preds is None: all_guids = guids.detach().cpu().numpy() preds = logits.detach().cpu().numpy() out_label_ids = inputs["labels"].detach().cpu().numpy() else: all_guids = np.append(all_guids, guids.detach().cpu().numpy(), axis=0) preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0) all_probs = softmax(preds, axis=1)[:,1] eval_loss = eval_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) elif args.output_mode == "regression": preds = np.squeeze(preds) result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) if args.cross_validation: cross_validation_results_dir = os.path.join(eval_output_dir, "cross_validation_results") if not os.path.exists(cross_validation_results_dir) and args.local_rank in [-1, 0]: os.makedirs(cross_validation_results_dir) output_eval_file = os.path.join(cross_validation_results_dir, "eval_results_fold_" + str(args.fold_number) +".txt") output_pred_file = os.path.join(cross_validation_results_dir, "submission_fold_" + str(args.fold_number) + ".csv") all_guids = all_guids.tolist() all_probs = all_probs.tolist() submission = pd.DataFrame() submission['sentenceID'] = all_guids submission['pred_prob'] = all_probs submission.to_csv(output_pred_file, index=False) else: output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") output_pred_file = os.path.join(eval_output_dir, prefix, "submission.csv") all_guids = all_guids.tolist() all_probs = all_probs.tolist() submission = pd.DataFrame() submission['sentenceID'] = all_guids submission['pred_prob'] = all_probs submission.to_csv(output_pred_file, index=False) # submission = pd.read_csv("submissions/sample_submission.csv") # submission = pd.read_csv("submissions/train_sample_submission.csv") # submission['pred_label'] = preds.reshape(len(preds), 1) # submission.to_csv(output_pred_file, index=False) with open(output_eval_file, "w") as writer: logger.info("***** Eval results {} *****".format(prefix)) for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) return results
def main(args): global HALT_filename, CHECKPOINT_tempfile # Create model directory & other aux folders for logging where_to_save = os.path.join(args.save_dir, args.dataset, args.model_name, args.image_model, args.experiment_name) checkpoints_dir = os.path.join(where_to_save, 'checkpoints') suffix = '_'.join([args.dataset, args.model_name, str(args.seed)]) checkpoint_filename = os.path.join(checkpoints_dir, '_'.join([suffix, 'checkpoint'])) print(checkpoint_filename) logs_dir = os.path.join(where_to_save, 'logs') tb_logs = os.path.join(where_to_save, 'tb_logs', args.dataset, args.model_name + '_' + str(args.seed)) make_dir(where_to_save) make_dir(logs_dir) make_dir(checkpoints_dir) make_dir(tb_logs) # Create loggers # stdout logger stdout_logger = logging.getLogger('STDOUT') stdout_logger.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(threadName)s - %(levelname)s: %(message)s') fh_out = logging.FileHandler( os.path.join(logs_dir, 'train_{}.log'.format(suffix))) fh_out.setFormatter(formatter) stdout_logger.addHandler(fh_out) ch = logging.StreamHandler(stream=sys.stdout) ch.setFormatter(formatter) stdout_logger.addHandler(ch) # stderr logger stderr_logger = logging.getLogger('STDERR') fh_err = logging.FileHandler(os.path.join(logs_dir, 'train_{}.err'.format(suffix)), mode='w') fh_err.setFormatter(formatter) stderr_logger.addHandler(fh_err) sl_stderr = StreamToLogger(stderr_logger, logging.ERROR) sys.stderr = sl_stderr # HALT file is used as a sign of job completion. # Check if no HALT file left from previous runs. HALT_filename = os.path.join(where_to_save, 'HALT_{}'.format(suffix)) if os.path.isfile(HALT_filename): os.remove(HALT_filename) # Remove CHECKPOINT_tempfile CHECKPOINT_tempfile = checkpoint_filename + '.tmp.ckpt' if os.path.isfile(CHECKPOINT_tempfile): os.remove(CHECKPOINT_tempfile) # Create tensorboard visualizer if args.tensorboard: logger = Visualizer(tb_logs, name='visual_results', resume=args.resume) # Check if we want to resume from last checkpoint of current model checkpoint = None if args.resume: if os.path.isfile(checkpoint_filename + '.ckpt'): checkpoint = torch.load(checkpoint_filename + '.ckpt', map_location=map_loc) num_epochs = args.num_epochs args = checkpoint['args'] args.num_epochs = num_epochs # Build data loader data_loaders = {} datasets = {} for split in ['train', 'val']: transforms_list = [transforms.Resize(args.image_size)] # Image pre-processing if split == 'train': transforms_list.append(transforms.RandomHorizontalFlip()) transforms_list.append( transforms.RandomAffine(degrees=10, translate=(0.1, 0.1))) transforms_list.append(transforms.RandomCrop(args.crop_size)) else: transforms_list.append(transforms.CenterCrop(args.crop_size)) transforms_list.append(transforms.ToTensor()) transforms_list.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) transform = transforms.Compose(transforms_list) # Load dataset path datapaths = json.load(open('../configs/datapaths.json')) dataset_root = datapaths[args.dataset] data_loaders[split], datasets[split] = get_loader( dataset=args.dataset, dataset_root=dataset_root, split=split, transform=transform, batch_size=args.batch_size, include_eos=(args.decoder != 'ff'), shuffle=(split == 'train'), num_workers=args.num_workers, drop_last=(split == 'train'), shuffle_labels=args.shuffle_labels, seed=args.seed, checkpoint=checkpoint) stdout_logger.info('Dataset {} split contains {} images'.format( split, len(datasets[split]))) vocab_size = len(datasets[split].get_vocab()) stdout_logger.info('Vocabulary size is {}'.format(vocab_size)) # Build the model model = get_model(args, vocab_size) # add model parameters if model.image_encoder.last_module is not None: params = list(model.decoder.parameters()) + list( model.image_encoder.last_module.parameters()) else: params = list(model.decoder.parameters()) params_cnn = list(model.image_encoder.pretrained_net.parameters()) n_p_cnn = sum(p.numel() for p in params_cnn if p.requires_grad) n_p = sum(p.numel() for p in params if p.requires_grad) total = n_p + n_p_cnn stdout_logger.info("CNN params: {}".format(n_p_cnn)) stdout_logger.info("decoder params: {}".format(n_p)) stdout_logger.info("total params: {}".format(total)) # encoder and decoder optimizers if params_cnn is not None and args.finetune_after == 0: optimizer = torch.optim.Adam( [{ 'params': params }, { 'params': params_cnn, 'lr': args.learning_rate * args.scale_learning_rate_cnn }], lr=args.learning_rate, weight_decay=args.weight_decay) keep_cnn_gradients = True stdout_logger.info("Fine tuning image encoder") else: optimizer = torch.optim.Adam(params, lr=args.learning_rate) keep_cnn_gradients = False stdout_logger.info("Freezing image encoder") # early stopping and checkpoint es_best = {'o_f1': 0, 'c_f1': 0, 'i_f1': 0, 'average': 0} epoch_best = {'o_f1': -1, 'c_f1': -1, 'i_f1': -1, 'average': -1} if checkpoint is not None: optimizer.load_state_dict(checkpoint['optimizer']) for state in optimizer.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): state[k] = v.to(device) model.load_state_dict(checkpoint['state_dict']) es_best = checkpoint['es_best'] epoch_best = checkpoint['epoch_best'] if device != 'cpu' and torch.cuda.device_count() > 1: model = nn.DataParallel(model) model = model.to(device) cudnn.benchmark = True if not hasattr(args, 'current_epoch'): args.current_epoch = 0 # Train the model decay_factor = 1.0 start_step = 0 if checkpoint is None else checkpoint['current_step'] curr_pat = 0 if checkpoint is None else checkpoint['current_pat'] for epoch in range(args.current_epoch, args.num_epochs): # save current epoch for resuming if args.tensorboard: logger.reset() # increase / decrease values for moving params if args.decay_lr: frac = epoch // args.lr_decay_every decay_factor = args.lr_decay_rate**frac new_lr = args.learning_rate * decay_factor stdout_logger.info('Epoch %d. lr: %.5f' % (epoch, new_lr)) set_lr(optimizer, decay_factor) if args.finetune_after != -1 and args.finetune_after < epoch \ and not keep_cnn_gradients and params_cnn is not None: stdout_logger.info("Starting to fine tune CNN") # start with learning rates as they were (if decayed during training) optimizer = torch.optim.Adam([{ 'params': params }, { 'params': params_cnn, 'lr': decay_factor * args.learning_rate * args.scale_learning_rate_cnn }], lr=decay_factor * args.learning_rate) keep_cnn_gradients = True for split in ['train', 'val']: if split == 'train': model.train() else: model.eval() total_step = len(data_loaders[split]) loader = iter(data_loaders[split]) total_loss_dict = { 'label_loss': [], 'eos_loss': [], 'cardinality_loss': [], 'loss': [], 'o_f1': [], 'c_f1': [], 'i_f1': [], } torch.cuda.synchronize() start = time.time() overall_error_counts = { 'tp_c': 0, 'fp_c': 0, 'fn_c': 0, 'tn_c': 0, 'tp_all': 0, 'fp_all': 0, 'fn_all': 0 } i = 0 if split == 'val' else start_step for info in loader: img_inputs, gt = info # adapt gts by adding pad_value to match maxnumlabel length gt = [ sublist + [vocab_size - 1] * (args.maxnumlabels - len(sublist)) for sublist in gt ] gt = torch.LongTensor(gt) # move to device img_inputs = img_inputs.to(device) gt = gt.to(device) loss_dict = {} if split == 'val': with torch.no_grad(): # get losses and label predictions _, predictions = model(img_inputs, maxnumlabels=args.maxnumlabels, compute_losses=False, compute_predictions=True) # convert model predictions and targets to k-hots pred_k_hots = label2_k_hots( predictions, vocab_size - 1, remove_eos=(args.decoder != 'ff')) target_k_hots = label2_k_hots( gt, vocab_size - 1, remove_eos=(args.decoder != 'ff')) # update overall and per class error types update_error_counts(overall_error_counts, pred_k_hots, target_k_hots) # update per image error types i_f1s = [] for i in range(pred_k_hots.size(0)): image_error_counts = { 'tp_c': 0, 'fp_c': 0, 'fn_c': 0, 'tn_c': 0, 'tp_all': 0, 'fp_all': 0, 'fn_all': 0 } update_error_counts(image_error_counts, pred_k_hots[i].unsqueeze(0), target_k_hots[i].unsqueeze(0)) image_metrics = compute_metrics( image_error_counts, which_metrics=['f1']) i_f1s.append(image_metrics['f1']) loss_dict['i_f1'] = np.mean(i_f1s) del predictions, pred_k_hots, target_k_hots, image_metrics else: losses, _ = model(img_inputs, gt, maxnumlabels=args.maxnumlabels, keep_cnn_gradients=keep_cnn_gradients, compute_losses=True) # label loss label_loss = losses['label_loss'] label_loss = label_loss.mean() loss_dict['label_loss'] = label_loss.item() # cardinality loss if args.pred_cardinality != 'none': cardinality_loss = losses['cardinality_loss'] cardinality_loss = cardinality_loss.mean() loss_dict['cardinality_loss'] = cardinality_loss.item() else: cardinality_loss = 0 # eos loss if args.perminv: eos_loss = losses['eos_loss'] eos_loss = eos_loss.mean() loss_dict['eos_loss'] = eos_loss.item() else: eos_loss = 0 # total loss loss = args.loss_weight[0] * label_loss \ + args.loss_weight[1]*cardinality_loss + \ args.loss_weight[2]*eos_loss loss_dict['loss'] = loss.item() # optimizer step model.zero_grad() loss.backward() optimizer.step() del loss, losses del img_inputs for key in loss_dict.keys(): total_loss_dict[key].append(loss_dict[key]) # Print log info if args.log_step != -1 and i % args.log_step == 0: elapsed_time = time.time() - start lossesstr = "" for k in total_loss_dict.keys(): if len(total_loss_dict[k]) == 0: continue this_one = "%s: %.4f" % ( k, np.mean(total_loss_dict[k][-args.log_step:])) lossesstr += this_one + ', ' # this only displays nll loss on captions, the rest of losses will # be in tensorboard logs strtoprint = 'Split: %s, Epoch [%d/%d], Step [%d/%d], Losses: %sTime: %.4f' % ( split, epoch, args.num_epochs, i, total_step, lossesstr, elapsed_time) stdout_logger.info(strtoprint) if args.tensorboard and split == 'train': logger.scalar_summary( mode=split + '_iter', epoch=total_step * epoch + i, **{ k: np.mean(v[-args.log_step:]) for k, v in total_loss_dict.items() if v }) torch.cuda.synchronize() start = time.time() i += 1 if split == 'train': increase_loader_epoch(data_loaders['train']) start_step = 0 if split == 'val': overal_metrics = compute_metrics(overall_error_counts, ['f1', 'c_f1'], weights=None) total_loss_dict['o_f1'] = overal_metrics['f1'] total_loss_dict['c_f1'] = overal_metrics['c_f1'] if args.tensorboard: # 1. Log scalar values (scalar summary) logger.scalar_summary( mode=split, epoch=epoch, **{ k: np.mean(v) for k, v in total_loss_dict.items() if v }) # early stopping metric_average = 0 best_at_checkpoint_metric = False if args.metric_to_checkpoint != 'average': es_value = np.mean(total_loss_dict[args.metric_to_checkpoint]) if es_value > es_best[args.metric_to_checkpoint]: es_best[args.metric_to_checkpoint] = es_value epoch_best[args.metric_to_checkpoint] = epoch best_at_checkpoint_metric = True save_checkpoint( model, optimizer, args, es_best, epoch_best, 0, 0, '{}.best.{}'.format(checkpoint_filename, args.metric_to_checkpoint)) else: for metric in ['o_f1', 'c_f1', 'i_f1']: es_value = np.mean(total_loss_dict[metric]) metric_average += es_value metric_average /= 3 if metric_average > es_best['average']: es_best['average'] = metric_average epoch_best['average'] = epoch if 'average' == args.metric_to_checkpoint: best_at_checkpoint_metric = True save_checkpoint( model, optimizer, args, es_best, epoch_best, 0, 0, '{}.best.average'.format(checkpoint_filename)) if best_at_checkpoint_metric: curr_pat = 0 else: curr_pat += 1 args.current_epoch = epoch + 1 # Save the epoch at which the model needs to start save_checkpoint(model, optimizer, args, es_best, epoch_best, 0, curr_pat, checkpoint_filename) stdout_logger.info('Saved checkpoint for epoch {}.'.format(epoch)) if curr_pat > args.patience: break # Mark job as finished f = open(HALT_filename, 'w') for metric in es_best.keys(): f.write('{}:{}\n'.format(metric, es_best[metric])) f.close() if args.tensorboard: logger.close()
def evaluate_model(Net, seeds, mini_batch_size=100, optimizer=optim.Adam, criterion=nn.CrossEntropyLoss(), n_epochs=40, eta=1e-3, lambda_l2=0, alpha=0.5, beta=0.5, plot=True, statistics=True, rotate=False, translate=False, swap_channel=False, GPU=False): """ General : 10 rounds of network training / validation with statistics - Repeat the training/validation procedure 10 times for ten different seeds in seeds 1) At every seed -> reinitializes a network and a dataset : train,validation and test 2) Weights initialization and data loading are using the seed 3) Record the train and validation accuracy and loss and can display their evolution curve 4) Compute the statistics at the end of each training for performance evaluation i) Mean training accuracy for each seed -> value at the end of the last epoch ii) Mean validation accuracy for each seed -> value at the end of the last epoch iii) Mean test accuracy for each seed -> compute the accuracy on the test after each training -> display a boxplot of the statistics if statistics is true and print the mean and standard deviation Input : - Net : A network dictionnary from the <Nets> class - seeds : a list of seed to iterate over for pseudo random number generator used in weight initialization and data loading -> mini_batch_size,optimizer, criterion, n_epochs, eta, lambda_2, alpha, beta see training.py - plot : if true plot the learning curve evolution over the epochs -> default true - statistics : if true display the boxplot of the train accuracies, validations and test and print the mean and standard deviation statistics -> rotate,translate and swap_channels -> data augmentation see loader.py Output : - train_result : A (10x4xn_epochs) tensor 10 -> seed 4 -> train loss ,train accuracy, validation loss, validation accuracy n_epochs -> evolution during training - test_losses : A tensor of shape (10,) containing the test loss at each seed - test_accuracies : A tensor of shape (10,) containing the test loss at each seed """ # tensor initialization to store the metrics train_results = torch.empty(len(seeds), 4, n_epochs) test_losses = [] test_accuracies = [] for n, seed in enumerate(seeds): # set the pytorch seed torch.manual_seed(seed) torch.cuda.manual_seed(seed) # set the seed for random spliting of the dataset in training and validation random.seed(0) # load the dataset train,validation and test data = PairSetMNIST() train_data = Training_set(data) test_data = Test_set(data) train_data_split = Training_set_split(train_data, rotate, translate, swap_channel) validation_data = Validation_set(train_data) # construct the net type with default parameter if (Net['net_type'] == 'Net2c'): model = Net['net'](nb_hidden=Net['hidden_layers'], dropout_prob=Net['drop_prob']) if (Net['net_type'] == 'LeNet_sharing'): model = Net['net'](nb_hidden=Net['hidden_layers'], dropout_ws=Net['drop_prob_ws'], dropout_comp=Net['drop_prob_comp']) if (Net['net_type'] == 'LeNet_sharing_aux'): # check if any data augmentation has been called # if none construct with tuned parameters without data augmentation # if yes construct with tuned parameters with data augmentation if (rotate == False and translate == False and swap_channel == False): model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'], nbhidden_comp=Net['hidden_layers_comp'], drop_prob_aux=Net['drop_prob_aux'], drop_prob_comp=Net['drop_prob_comp']) else: Net['learning rate'] = Net['learning rate augm'] model = Net['net'](nbhidden_aux=Net['hidden_layers_aux'], nbhidden_comp=Net['hidden_layers_comp'], drop_prob_aux=Net['drop_prob_aux_augm'], drop_prob_comp=Net['drop_prob_comp_augm']) if (Net['net_type'] == 'Google_Net'): model = Net['net'](channels_1x1=Net['channels_1x1'], channels_3x3=Net['channels_3x3'], channels_5x5=Net['channels_5x5'], pool_channels=Net['pool_channels'], nhidden=Net['hidden_layers'], drop_prob_comp=Net['drop_prob_comp'], drop_prob_aux=Net['drop_prob_aux']) if GPU and cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') model = model.to(device) # train the model on the train set and validate at each epoch train_losses, train_acc, valid_losses, valid_acc = train_model( model, train_data_split, validation_data, device, mini_batch_size, optimizer, criterion, n_epochs, Net['learning rate'], lambda_l2, alpha, beta) # store the training and validation accuracies and losses during the training train_results[n, ] = torch.tensor( [train_losses, train_acc, valid_losses, valid_acc]) # compute the loss and accuracy of the model on the test set test_loss, test_acc = compute_metrics(model, test_data, device) # store the test metrics in the list test_losses.append(test_loss) test_accuracies.append(test_acc) # learning curve if plot: learning_curve(train_losses, train_acc, valid_losses, valid_acc) print( 'Seed {:d} | Test Loss: {:.4f} | Test Accuracy: {:.2f}%\n'.format( n, test_loss, test_acc)) # store the train, validation and test accuracies in a tensor for the boxplot data = torch.stack([ train_results[:, 1, (n_epochs - 1)], train_results[:, 3, (n_epochs - 1)], torch.tensor(test_accuracies) ]) data = data.view(1, 3, 10) # boxplot if statistics: Title = " Models accuracies" models = [Net['net_type']] boxplot(data, Title, models, True) return train_results, torch.tensor(test_losses), torch.tensor( test_accuracies)