def summarize_probability(prob_dict, action_to_idx, title): acc = [AverageMeter(),AverageMeter()] stat = {} for vname, item in tqdm(prob_dict.items(), total=len(prob_dict)): try: action_name = vname.split('/')[-3] except: action_name = vname.split('/')[-2] target = action_to_idx(action_name) mean_prob = torch.stack(item['mean_prob'], 0).mean(0) mean_top1, mean_top5 = calc_topk_accuracy(mean_prob, torch.LongTensor([target]).cuda(), (1,5)) stat[vname] = {'mean_prob': mean_prob.tolist()} acc[0].update(mean_top1.item(), 1) acc[1].update(mean_top5.item(), 1) print('Mean: Acc@1: {acc[0].avg:.4f} Acc@5: {acc[1].avg:.4f}' .format(acc=acc)) with open(os.path.join(os.path.dirname(args.test), '%s-prob-%s.json' % (os.path.basename(args.test), title)), 'w') as fp: json.dump(stat, fp) return acc
def train(train_loader, model, criterion, optimizer, epoch, print_freq=1000): # amp_handle = amp.init() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() model = model.train() end = time.time() print("Start Training") epoch_time = time.time() for i, (imgs, caps, lengths) in enumerate(train_loader): print("%2.2f" % (i / len(train_loader) * 100), "\%", end="\r") data_time.update(time.time() - end) optimizer.zero_grad() output_imgs, output_caps = model(imgs, caps, lengths) loss = criterion(output_imgs, output_caps) loss.backward() optimizer.step() losses.update(loss.item(), imgs.size(0)) batch_time.update(time.time() - end) end = time.time() if i % print_freq == 0 or i == (len(train_loader) - 1): print("Epoch: [{0}][{1}/{2}]\t" "Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t" "Data {data_time.val:.3f} ({data_time.avg:.3f})\t" "Loss {loss.val:.4f} ({loss.avg:.4f})".format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, )) print("Epoch time :", time.time() - epoch_time) return losses.avg, batch_time.avg, data_time.avg
def do_eval(opt, epoch, model, DatasetFactory, logger, best): # Based this code on test.py's non-prefetched code path: Detector = detector_factory[opt.task] dataset = DatasetFactory(opt, "val") detector = Detector(opt, model) results = {} num_iters = len(dataset) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = dataset.images[ind] img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_path = os.path.join(dataset.img_dir, img_info['file_name']) if opt.task == 'ddd': ret = detector.run(img_path, img_info['calib']) else: ret = detector.run(img_path) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {:.3f} '.format(t, avg_time_stats[t].avg) bar.next() bar.finish() metric = float("-inf") # Capture metric of interest, e.g., for COCO eval, something like AP50: eval_stats = dataset.run_eval(results, opt.save_dir, logger) if uses_coco_eval(opt): ap50 = eval_stats[1] ap25 = eval_stats[12] metric = ap25 # Log results to log.txt and/or tensorboard: logger.scalar_summary("val_ap50", ap50, epoch) logger.scalar_summary("val_ap25", ap25, epoch) else: # Pascal VOC: metric = eval_stats["Mean AP"] # Log results to log.txt and/or tensorboard: logger.scalar_summary("mean_AP", metric, epoch) # Best model checkpointing: if metric > best: best = metric save_model( os.path.join(opt.save_dir, "model_best.pth"), epoch, model ) return best
def eval_one_epoch(model, loader, criterion, epoch, logger, args): losses = AverageMeter("Loss") y_true, y_pred = [], [] model.eval() with torch.no_grad(): for batch_idx, (data, target, idx) in enumerate(loader): data = data.cuda() target = target.cuda() z, logits = model(data) loss = criterion(logits, target.view(-1)) losses.update(loss.item(), data.shape[0]) probabilities = nn.Softmax(dim=1)(logits) _, predictions = torch.max(probabilities, 1) y_pred.append(predictions.cpu().numpy().reshape(-1)) y_true.append(target.cpu().numpy().reshape(-1)) # append invalid samples at the beginning of the test sequence if loader.dataset.prefix == "test": ws = data.shape[1] - 1 samples_invalid = [y_true[0][0]] * ws y_true.append(samples_invalid) y_pred.append(samples_invalid) y_true = np.concatenate(y_true, 0) y_pred = np.concatenate(y_pred, 0) acc = 100.0 * metrics.accuracy_score(y_true, y_pred) fm = 100.0 * metrics.f1_score(y_true, y_pred, average="macro") fw = 100.0 * metrics.f1_score(y_true, y_pred, average="weighted") if logger: logger.add_scalars("Loss", {"CrossEntropy": losses.avg}, epoch) logger.add_scalar("Acc", acc, epoch) logger.add_scalar("Fm", fm, epoch) logger.add_scalar("Fw", fw, epoch) if epoch % 50 == 0 or not args.train_mode: plot_confusion( y_true, y_pred, os.path.join(model.path_visuals, f"cm/{loader.dataset.prefix}"), epoch, class_map=args.class_map, ) return losses.avg, acc, fm, fw
def test(opt): os.environ["CUDA_VISIBLE_DEVICES"] = opt.gpus_str Dataset = dataset_factory[opt.test_dataset] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) Logger(opt) split = "val" if not opt.trainval else "test" dataset = Dataset(opt, split) detector = Detector(opt) if opt.load_results != "": # load results in json load_results = json.load(open(opt.load_results, "r")) results = {} num_iters = len(dataset) if opt.num_iters < 0 else opt.num_iters bar = Bar("{}".format(opt.exp_id), max=num_iters) time_stats = ["tot", "load", "pre", "net", "dec", "post", "merge"] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = dataset.images[ind] img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_path = os.path.join(dataset.img_dir, img_info["file_name"]) input_meta = {} if "calib" in img_info: input_meta["calib"] = img_info["calib"] if opt.tracking and ("frame_id" in img_info) and img_info["frame_id"] == 1: detector.reset_tracking() input_meta["pre_dets"] = load_results[img_id] ret = detector.run(img_path, input_meta) results[img_id] = ret["results"] Bar.suffix = "[{0}/{1}]|Tot: {total:} |ETA: {eta:} ".format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td ) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + "|{} {:.3f} ".format(t, avg_time_stats[t].avg) bar.next() bar.finish() if opt.save_results: print( "saving results to", opt.save_dir + "/save_results_{}{}.json".format(opt.test_dataset, opt.dataset_version) ) json.dump( _to_list(copy.deepcopy(results)), open(opt.save_dir + "/save_results_{}{}.json".format(opt.test_dataset, opt.dataset_version), "w"), ) dataset.run_eval(results, opt.save_dir)
def eval_model(epoch, model, loader): global CURRENT_ACC model.eval() model.to(device) positive_distance_meter = AverageMeter() negative_distance_meter = AverageMeter() loss_meter = AverageMeter() acc_meter = AverageMeter() print('=' * 20 + "Model Eval" + '=' * 20) for i, batch in tqdm(enumerate(loader)): start = time.time() with torch.no_grad(): loss, acc, positive_distance, negative_distance = batch_eval(batch, model) loss_meter.update(loss.item()) acc_meter.update(acc.item()) positive_distance_meter.update(positive_distance.mean().item()) negative_distance_meter.update(negative_distance.mean().item()) end = time.time() used_time = end - start if (i) % (args.log_every / 2) == 0: display = 'epoch=' + str(epoch) + \ '\tglobal_step=%d' % (GLOBAL_STEP) + \ '\tloss=%.4f' % (loss_meter.val) + \ '\tloss_avg=%.4f' % (loss_meter.avg) + \ '\tpos_avg=%.4f' % (positive_distance_meter.avg) + \ '\tneg_avg=%.4f' % (negative_distance_meter.avg) + \ '\tacc=%.4f' % (acc_meter.avg) + \ '\ttime=%.2fit/s' % (1. / used_time) tqdm.write(display) print("Final Acc: %.6f" % (acc_meter.avg)) print("Final Loss Acc: %.6f" % (loss_meter.avg)) print("Final Positive Distance: %.6f" % (positive_distance_meter.avg)) print("Final Negative Distance: %.6f" % (negative_distance_meter.avg)) CURRENT_ACC = acc_meter.avg return
def validate(data_loader, model, criterion, device, epoch, args): losses = AverageMeter() accuracy = AverageMeter() model.eval() with torch.no_grad(): for idx, (input_seq, target) in tqdm(enumerate(data_loader), total=len(data_loader)): input_seq = input_seq.to(device) target = target.to(device) B = input_seq.size(0) output, _ = model(input_seq) #[_, N, D] = output.size() #output = output.view(B*N, D) #if len(target.shape) == 1: # target = target.repeat(1, N).view(-1) # metric = "Acc" #else: # target = target.repeat(1, N).view(-1, D).float() # metric = "mAP" target = target.float() metric = "mAP" loss = criterion(output, target) acc = calc_accuracy(output, target) losses.update(loss.item(), B) accuracy.update(acc.item(), B) print('Loss {loss.avg:.4f}\t' '{metric}: {acc.avg:.4f} \t'.format(loss=losses, acc=accuracy, metric=metric)) args.writer_val.add_scalar('global/loss', losses.avg, epoch) args.writer_val.add_scalar('global/accuracy', accuracy.avg, epoch) return losses.avg, accuracy.avg
def validate(val_loader, model): """Perform validation on the validation set""" batch_time = AverageMeter() top1 = AverageMeter() model.eval() end = time.time() val_size = len(val_loader) val_loader_iter = iter(val_loader) bar_format = '{desc}[{elapsed}<{remaining},{rate_fmt}]' pbar = tqdm(range(val_size), file=sys.stdout, bar_format=bar_format, ncols=140) with torch.no_grad(): for idx_iter in pbar: input, label = next(val_loader_iter) input = input.cuda() label = label.cuda() # compute output output = torch.sigmoid(model(input, task='new')[0]) output = (output + torch.sigmoid(model(torch.flip(input, dims=(3,)), task='new')[0])) / 2 # accumulate accuracyk prec1, gt_num = accuracy(output.data, label, args.num_class, topk=(1,)) top1.update(prec1[0], gt_num[0]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() description = "[Acc@1-mean: %.2f][Acc@1-cls: %s]"%(top1.vec2sca_avg, str(top1.avg.numpy().round(1))) pbar.set_description("[Step %d/%d]"%(idx_iter + 1, val_size) + description) logging.info(' * Prec@1 {top1.vec2sca_avg:.3f}'.format(top1=top1)) logging.info(' * Prec@1 {top1.avg}'.format(top1=top1)) return top1.vec2sca_avg
def __init__(self, config, data_loader): self.device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") self.in_channels = config.in_channels self.out_channels = config.out_channels self.nf = config.nf self.storing_channels = config.storing_channels self.lr = config.lr self.b1 = config.b1 self.b2 = config.b2 self.weight_decay = config.weight_decay self.decay_epoch = config.decay_epoch self.content_loss_factor = config.content_loss_factor self.perceptual_loss_factor = config.perceptual_loss_factor self.generator_loss_factor = config.generator_loss_factor self.discriminator_loss_factor = config.discriminator_loss_factor self.penalty_loss_factor = config.penalty_loss_factor self.eta = config.eta self.checkpoint_dir = config.checkpoint_dir self.sample_dir = config.sample_dir self.epoch = config.epoch self.num_epoch = config.num_epoch self.image_size = config.image_size self.data_loader = data_loader self.content_losses = AverageMeter() self.generator_losses = AverageMeter() self.perceptual_losses = AverageMeter() self.discriminator_losses = AverageMeter() self.ae_losses = AverageMeter() self.visdom = Visdom() self.build_model()
def evaluate(loader, model): print("Evaluate") # Set model to eval model.eval() accuracy = AverageMeter() positive_accuracy = AverageMeter() negative_accuracy = AverageMeter() with torch.no_grad(): for batch_idx, (x, y) in enumerate(loader): x = x.to(device=device) y = y.to(device=device).to(torch.float32) y = torch.unsqueeze(y, 1) scores = model(x) loss = criterion(scores, y) scores = torch.squeeze(scores, 1) y = torch.squeeze(y, 1) acc = get_accuracy(y, scores) neg_acc, pos_acc = get_accuracy_per_class(y.cpu(), scores.cpu()) accuracy.update(acc) positive_accuracy.update(pos_acc) negative_accuracy.update(neg_acc) wandb.log({ "valid_acc": accuracy.avg, "positive_acc": positive_accuracy.avg, "negative_acc": negative_accuracy.avg, "valid_loss": loss.item() }) # Set model back to train model.train()
def _valid_epoch(self): valid_loss = AverageMeter() valid_probs = [] for step, batch in enumerate(self.valid_loader): self.model.eval() batch = tuple(t.to(self.device) for t in batch) batch_size = batch[1].size(0) with torch.no_grad(): op = batch[0] inputs = { "input_ids_a": batch[1], "token_type_ids_a": batch[2], "attention_mask_a": batch[3], "input_ids_b": batch[4], "token_type_ids_b": batch[5], "attention_mask_b": batch[6], "input_ids_c": batch[7], "token_type_ids_c": batch[8], "attention_mask_c": batch[9], } if self.fts_flag: inputs.update({ "x_a": batch[10], "x_b": batch[11], "x_c": batch[12] }) anchor, positive, negative = self.model(**inputs) # loss = self.criterion(anchor, positive, negative) loss = self.criterion(op.float(), anchor, positive, negative) valid_loss.update(loss.item(), batch_size) anchor = anchor.to("cpu").numpy() positive = positive.to("cpu").numpy() negative = negative.to("cpu").numpy() pos_dist = np.sqrt( np.sum(np.square(anchor - positive), axis=-1, keepdims=True)) neg_dist = np.sqrt( np.sum(np.square(anchor - negative), axis=-1, keepdims=True)) probs = pos_dist - neg_dist # probs = (op.to("cpu").numpy() * (pos_dist - neg_dist)).diagonal() valid_probs.append(probs) valid_probs = np.concatenate(valid_probs) valid_log = {"val_loss": valid_loss.avg, "val_probs": valid_probs} return valid_log
def test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str Dataset = dataset_factory[opt.dataset] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) Logger(opt) Detector = detector_factory[opt.task] split = 'val' if (not opt.trainval) and (not opt.test) else 'test' dataset = Dataset(opt, split) detector = Detector(opt) results = {} num_iters = len(dataset) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = dataset.images[ind] img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_path = os.path.join(dataset.img_dir, img_info['file_name']) n_people = len(dataset.coco.getAnnIds(imgIds=[img_id])) if n_people not in times_per_people.keys(): times_per_people[n_people] = {stat: 0 for stat in time_stats} if opt.task == 'ddd': ret = detector.run(img_path, img_info['calib']) else: ret = detector.run(img_path) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {:.3f} '.format( t, avg_time_stats[t].avg) times_per_people[n_people][t] += avg_time_stats[t].avg bar.next() bar.finish() dataset.run_eval(results, opt.save_dir) if opt.save_times_per_people: out_dir_time = "./time_reports/{}".format(opt.exp_id) if not os.path.isdir(out_dir_time): os.mkdir(out_dir_time) with open("{}/times_per_people.json".format(out_dir_time)) as f: json.dump(times_per_people, f)
def __init__(self, sampler, frames_threshold): """ @sampler: will mostly be an instance of DistributedSampler. Though it should work with any sampler. @frames_threshold: maximum area of the batch """ self.sampler = sampler self.frames_threshold = frames_threshold # indices = [(idx1, #frames1), (idx2, #frames2), ...] # batches = [[idx1, idx2, idx3], [idx4], [idx5, idx6], ...] indices, batches = list(), list() # the dataset to which these indices are pointing to dataset = self.sampler.dataset # get all the indices and corresponding durations from # the sampler for idx in self.sampler: indices.append((idx, dataset.utt_list[idx]["mel_frame"])) # sort the indices according to duration indices.sort(key=lambda elem: int(elem[1])) # make sure that you will be able to serve all the utterances assert indices[-1][1] < self.frames_threshold, ( "Won't be able" "to serve all sequences. frames_threshold={} while longest" " utterance has {} frames").format(self.frames_threshold, indices[-1][1]) # start clubbing the utterances together batch = list() batch_frames, batch_area = 0, 0 average_meter = AverageMeter('Padding Efficiency') for idx, frames in indices: # consider adding this utterance to the current batch if batch_frames + frames <= self.frames_threshold: # can add to the batch batch.append(idx) batch_frames += frames batch_area = frames * len(batch) # print('idx=',idx) # print('frames=',frames) # print('batch_area=',batch_area) else: # log the stats and add previous batch to batches average_meter.add(batch_frames, batch_area) batches.append(batch) # make a new one batch = list([idx]) batch_frames, batch_area = frames, frames # don't need the 'indices' any more del indices self.batches = batches average_meter.display_results()
def valid(model_R, val_loader): error_names = ['valid loss'] errors = AverageMeter(i=len(error_names)) model_R.eval() with torch.no_grad(): for i, (left_img, right_img) in enumerate(val_loader): left_img = left_img.to(device) right_img = right_img.to(device) disp_pre = model_R(left_img, right_img) error = Img_warp_loss(disp_pre, left_img, right_img) errors.update([error.item()], args.batch_size) return errors.avg, error_names
def infer(self): self.load_model_for_eval() avg_test_loss = AverageMeter() avg_test_acc = AverageMeter() text_avg_test_acc = AverageMeter() combined_avg_test_acc = AverageMeter() for batch_idx, batch in enumerate(tqdm(self.test_loader)): # Get the model output and update the meters output = self.compute_loss(batch) avg_test_acc.update(output['correct'].cpu().numpy()) text_avg_test_acc.update(output['text_correct'].cpu().numpy()) combined_avg_test_acc.update( output['combined_correct'].cpu().numpy()) avg_test_loss.update([output['loss']]) print( 'Final test acc (audio) = {:.4f}, final test acc (text) = {:.4f}, final test acc (combined system) = {:.4f}, test loss = {:.4f}' .format(avg_test_acc.get(), text_avg_test_acc.get(), combined_avg_test_acc.get(), avg_test_loss.get())) return avg_test_loss.get(), avg_test_acc.get(), text_avg_test_acc.get()
def infer(self): self.load_model_for_eval() avg_test_loss = AverageMeter() avg_test_acc = AverageMeter() all_true_labels = [] all_pred_labels = [] all_audio_embeddings = [] all_text_embeddings = [] for batch_idx, batch in enumerate(tqdm(self.test_loader)): # Get the model output and update the meters output = self.compute_loss(batch) avg_test_acc.update(output['correct'].cpu().numpy()) avg_test_loss.update([output['loss']]) # Store the Predictions all_true_labels.append(batch['label'].cpu()) all_pred_labels.append(output['predicted'].cpu()) all_audio_embeddings.append(output['model_output']['audio_embed'].cpu()) all_text_embeddings.append(output['model_output']['text_embed'].cpu()) # Collect the predictions and embeddings for the full set all_true_labels = torch.cat(all_true_labels).numpy() all_pred_labels = torch.cat(all_pred_labels).numpy() all_audio_embeddings = torch.cat(all_audio_embeddings).numpy() all_text_embeddings = torch.cat(all_text_embeddings).numpy() # Save the embeddings and plot the confusion matrix np.savez_compressed('embeddings.npz', audio=all_audio_embeddings, text=all_text_embeddings, labels=all_true_labels) cm = confusion_matrix(all_true_labels, all_pred_labels) plot_confusion_matrix(cm, self.test_loader.dataset.labels_list(), normalize=True) print('Final test acc = {:.4f}, test loss = {:.4f}'.format(avg_test_acc.get(), avg_test_loss.get())) return avg_test_loss.get(), avg_test_acc.get()
def validate(data_loader, model, criterion, device, epoch, args): losses = AverageMeter() accuracy = [ [AverageMeter(), AverageMeter()], # forward top1, top5 [AverageMeter(), AverageMeter()] ] # backward top1, top5 model.eval() with torch.no_grad(): for idx, input_seq in enumerate(data_loader): input_seq = input_seq.to(device) B = input_seq.size(0) loss, loss_step, acc, extra = model(input_seq) for i in range(2): top1, top5 = acc[i].mean(0) # average acc across multi-gpus accuracy[i][0].update(top1.item(), B) accuracy[i][1].update(top5.item(), B) loss = loss.mean() # average loss across multi-gpus losses.update(loss.item(), B) print('Epoch: [{0}/{1}]\t' 'Loss {loss.val:.6f}\t' 'Acc: {acc[0][0].val:.4f}\t'.format(epoch, args.epochs, loss=losses, acc=accuracy)) args.writer_val.add_scalar('global/loss', losses.avg, epoch) args.writer_val.add_scalar('global/F-top1', accuracy[0][0].avg, epoch) args.writer_val.add_scalar('global/F-top5', accuracy[0][1].avg, epoch) args.writer_val.add_scalar('global/B-top1', accuracy[1][0].avg, epoch) args.writer_val.add_scalar('global/B-top5', accuracy[1][1].avg, epoch) return losses.avg, np.mean([accuracy[0][0].avg, accuracy[1][0].avg])
def train_epoch(self, data): tr_loss = AverageMeter() self.epoch_reset() update=0 for step, batch in enumerate(data): self.model.train() input_ids,src_len, input_mask, segment_ids, src_token,label_ids,tgt_len,tgt_token = batch input_ids = input_ids.to(self.device) input_mask = input_mask.to(self.device) segment_ids = segment_ids.to(self.device) label_ids = label_ids.to(self.device) logits = self.model(input_ids, input_mask, segment_ids) loss = self.criterion(output=logits, target=label_ids) if len(self.n_gpu) >= 2: loss = loss.mean() if self.gradient_accumulation_steps > 1: loss = loss / self.gradient_accumulation_steps else: loss.backward() clip_grad_norm_(self.model.parameters(), self.grad_clip) if (step + 1) % self.gradient_accumulation_steps == 0: self.optimizer.step() self.optimizer.zero_grad() self.global_step += 1 tr_loss.update(loss.item(), n=1) #存储训练过程中的输出和目标值 self.outputs.append(logits.cpu().detach()) self.targets.append(label_ids.cpu().detach()) update+=1 progress_bar(update, self.config.train_nbatchs) print("\n------------- train result --------------") # epoch metric self.outputs = torch.cat(self.outputs, dim=0).cpu().detach() self.targets = torch.cat(self.targets, dim=0).cpu().detach() self.result['loss'] = tr_loss.avg if self.epoch_metrics: for metric in self.epoch_metrics: metric(logits=self.outputs, target=self.targets) value = metric.value() if value: self.result[f'{metric.name()}'] = value if "cuda" in str(self.device): torch.cuda.empty_cache() return self.result
def test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str Dataset = dataset_factory[opt.dataset[0]] Dataset2 = dataset_factory[opt.dataset[1]] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) Logger(opt) Detector = detector_factory[opt.task] split = opt.test_dataset dataset = Dataset(opt, split) dataset2 = Dataset2(opt, split) detector = Detector(opt) results = {} num_iters = len(dataset) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = dataset.images[ind] img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_path = os.path.join(dataset.img_dir, img_info['file_name']) if detector.model._name == 'fusion': img_path2 = os.path.join(dataset2.img_dir, img_info['file_name']) if opt.task == 'ddd': ret = detector.run(img_path, img_info['calib']) else: if detector.model._name == 'fusion': ret = detector.run(img_path, img_path2) else: ret = detector.run(img_path) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {:.3f} '.format( t, avg_time_stats[t].avg) bar.next() bar.finish() dataset.run_eval(results, opt.save_dir)
def test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str Dataset = dataset_factory[opt.dataset] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) Logger(opt) Detector = detector_factory[opt.task] split = 'val' if not opt.trainval else 'test' dataset = Dataset(opt, split) detector = Detector(opt) if opt.task == 'car_pose_6dof': # pass loaded 3D models for debug visualisations detector.set_models(dataset.models) results = {} num_iters = len(dataset) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = dataset.images[ind] if opt.task == 'car_pose_6dof': img_path = os.path.join(dataset.img_dir, img_id + '.jpg') ret = detector.run(img_path, img_id, dataset.calib) else: img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_info['file_name'] img_path = os.path.join(dataset.img_dir, img_info['file_name']) img_name = img_info['file_name'][:-4] if opt.task == 'ddd': ret = detector.run(img_path, img_name, img_info['calib']) else: ret = detector.run(img_path, img_name) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {:.3f} '.format( t, avg_time_stats[t].avg) bar.next() bar.finish() dataset.run_eval(results, opt.save_dir)
def validate(config, testloader, model, writer_dict, device): rank = get_rank() world_size = get_world_size() model.eval() ave_loss = AverageMeter() tot_inter = np.zeros(config.DATASET.NUM_CLASSES) tot_union = np.zeros(config.DATASET.NUM_CLASSES) with torch.no_grad(): for i_iter, batch in enumerate(testloader): image, label, _, _ = batch size = label.size() label = label.long().to(device) image = image.to(device) loss, pred = model(image, label) if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]: pred = F.interpolate(pred, size=(size[-2], size[-1]), mode='bilinear', align_corners=False) reduced_loss = reduce_tensor(loss) ave_loss.update(reduced_loss.item()) batch_inter, batch_union = batch_intersection_union( pred, label, config.DATASET.NUM_CLASSES) tot_inter += batch_inter tot_union += batch_union if i_iter % config.PRINT_FREQ == 0 and rank == 0: msg = f'Iter: {i_iter}, Loss: {ave_loss.average() / world_size:.6f}' logging.info(msg) tot_inter = torch.from_numpy(tot_inter).to(device) tot_union = torch.from_numpy(tot_union).to(device) tot_inter = reduce_tensor(tot_inter).cpu().numpy() tot_union = reduce_tensor(tot_union).cpu().numpy() IoU = np.float64(1.0) * tot_inter / (np.spacing(1, dtype=np.float64) + tot_union) mean_IoU = IoU.mean() print_loss = ave_loss.average() / world_size if rank == 0: writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', print_loss, global_steps) writer.add_scalar('valid_mIoU', mean_IoU, global_steps) writer_dict['valid_global_steps'] = global_steps + 1 return print_loss, mean_IoU
def test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str Dataset = dataset_factory[opt.dataset] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) Logger(opt) Detector = detector_factory[opt.task] split = 'val' if not opt.trainval else 'test' dataset = Dataset(opt, split) detector = Detector(opt) results = {} num_iters = len(dataset) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} f = open("fps_test.txt", 'w') #fps_write for ind in range(num_iters): img_id = dataset.images[ind] img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_path = os.path.join(dataset.img_dir, img_info['file_name']) print(img_path) #print fps start_time = time.time() if opt.task == 'ddd': ret = detector.run(img_path, img_info['calib']) else: ret = detector.run(img_path) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {:.3f} '.format( t, avg_time_stats[t].avg) bar.next() f.write(str(1.0 / (time.time() - start_time)) + '\n') # fps_write bar.finish() dataset.run_eval(results, opt.save_dir)
def contrastive_evaluate(val_loader, model, memory_bank): top1 = AverageMeter('Acc@1', ':6.2f') model.eval() for batch in val_loader: images = batch['image'].cuda(non_blocking=True) target = batch['target'].cuda(non_blocking=True) output = model(images) output = memory_bank.weighted_knn(output) acc1 = 100 * torch.mean(torch.eq(output, target).float()) top1.update(acc1.item(), images.size(0)) return top1.avg
def train(model, data_loader, optimizer, lr_scheduler, epoch, LOG): stages = 4 losses = [AverageMeter() for _ in range(stages)] length_loader = len(data_loader) model.train() for batch_id, data in enumerate(data_loader()): left_img, right_img, gt = data mask = paddle.to_tensor(gt.numpy() > 0) gt_mask = paddle.masked_select(gt, mask) outputs = model(left_img, right_img) outputs = [paddle.squeeze(output) for output in outputs] tem_stage_loss = [] for index in range(stages): temp_loss = args.loss_weights[index] * F.smooth_l1_loss( paddle.masked_select(outputs[index], mask), gt_mask, reduction='mean') tem_stage_loss.append(temp_loss) losses[index].update( float(temp_loss.numpy() / args.loss_weights[index])) sum_loss = paddle.add_n(tem_stage_loss) sum_loss.backward() optimizer.step() optimizer.clear_grad() if batch_id % 5 == 0: info_str = [ 'Stage {} = {:.2f}({:.2f})'.format(x, losses[x].val, losses[x].avg) for x in range(stages) ] info_str = '\t'.join(info_str) info_str = 'Train Epoch{} [{}/{}] lr:{:.5f}\t{}'.format( epoch, batch_id, length_loader, optimizer.get_lr(), info_str) LOG.info(info_str) lr_scheduler.step() info_str = '\t'.join( ['Stage {} = {:.2f}'.format(x, losses[x].avg) for x in range(stages)]) LOG.info('Average train loss: ' + info_str)
def test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str Dataset = dataset_factory[opt.test_dataset] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) #Logger(opt) split = 'val' if not opt.trainval else 'test' dataset = Dataset(opt, split) detector = Detector(opt) if opt.load_results != '': # load results in json load_results = json.load(open(opt.load_results, 'r')) results = {} num_iters = len(dataset) if opt.num_iters < 0 else opt.num_iters bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind in range(num_iters): img_id = dataset.images[ind] img_info = dataset.coco.loadImgs(ids=[img_id])[0] img_path = os.path.join(dataset.img_dir, img_info['file_name']) input_meta = {} if 'calib' in img_info: input_meta['calib'] = img_info['calib'] if (opt.tracking and ('frame_id' in img_info) and img_info['frame_id'] == 1): detector.reset_tracking() input_meta['pre_dets'] = load_results[img_id] ret = detector.run(img_path, input_meta) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix = Bar.suffix + '|{} {:.3f} '.format(t, avg_time_stats[t].avg) bar.next() bar.finish() if opt.save_results: print('saving results to', opt.save_dir + '/save_results_{}{}.json'.format( opt.test_dataset, opt.dataset_version)) json.dump(_to_list(copy.deepcopy(results)), open(opt.save_dir + '/save_results_{}{}.json'.format( opt.test_dataset, opt.dataset_version), 'w')) dataset.run_eval(results, opt.save_dir)
def validate(config, testloader, model, writer_dict, device): rank = get_rank() world_size = get_world_size() model.eval() ave_loss = AverageMeter() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for _, batch in enumerate(testloader): image, label, _, _ = batch size = label.size() image = image.to(device) label = label.long().to(device) losses, pred = model(image, label) pred = F.upsample(input=pred, size=( size[-2], size[-1]), mode='bilinear') loss = losses.mean() reduced_loss = reduce_tensor(loss) ave_loss.update(reduced_loss.item()) confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) confusion_matrix = torch.from_numpy(confusion_matrix).to(device) reduced_confusion_matrix = reduce_tensor(confusion_matrix) confusion_matrix = reduced_confusion_matrix.cpu().numpy() pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() print_loss = ave_loss.average()/world_size if rank == 0: writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', print_loss, global_steps) writer.add_scalar('valid_mIoU', mean_IoU, global_steps) writer_dict['valid_global_steps'] = global_steps + 1 return print_loss, mean_IoU, IoU_array
def train(gc_net, optimizer): loss_names = ['loss train', 'acc train'] losses = AverageMeter(i=len(loss_names), precision=4) gc_net.train() optimizer.zero_grad() output = gc_net(features, adj) loss_train = F.nll_loss(output[idx_train], labels[idx_train]) acc_train = accuracy(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() losses.update([loss_train.item(), acc_train.item()]) return losses.avg, loss_names
def update_states(self, states, batch_size=1): if len(self.states) == 0: state_names = states.keys() self.states = OrderedDict( [(key, 0) for key in state_names] ) self.average_meters = OrderedDict( [(key, AverageMeter()) for key in state_names] ) self.states.update(states) for key, meter in self.average_meters.items(): meter.update(self.states[key], batch_size)
def train(epoch): # define meters loss_meter = AverageMeter() # put model into training mode model.train() for param_group in optimizer.param_groups: print('learning rate: {}'.format(param_group['lr'])) for i, sample in enumerate(tqdm(train_dataset_it)): im = sample['image'] instances = sample['instance'].squeeze() class_labels = sample['label'].squeeze() output = model(im) loss = criterion(output, instances, class_labels, **args['loss_w']) loss = loss.mean() optimizer.zero_grad() loss.backward() optimizer.step() if args['display'] and i % args['display_it'] == 0: with torch.no_grad(): visualizer.display(im[0], 'image') predictions = cluster.cluster_with_gt( output[0], instances[0], n_sigma=args['loss_opts']['n_sigma']) visualizer.display([predictions.cpu(), instances[0].cpu()], 'pred') sigma = output[0][2].cpu() sigma = (sigma - sigma.min()) / (sigma.max() - sigma.min()) sigma[instances[0] == 0] = 0 visualizer.display(sigma, 'sigma') seed = torch.sigmoid(output[0][3]).cpu() visualizer.display(seed, 'seed') loss_meter.update(loss.item()) return loss_meter.avg
def prefetch_test(opt): os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str Dataset = dataset_factory[opt.dataset] opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) loger = Logger(opt) Detector = detector_factory[opt.task] split = 'val' if not opt.test else 'test' dataset = Dataset(opt, split) detector = Detector(opt) data_loader = torch.utils.data.DataLoader( PrefetchDataset(opt, dataset, detector.pre_process), batch_size=1, shuffle=False, num_workers=0, pin_memory=True, collate_fn=test_collate_fn ) results = {} num_iters = len(dataset) bar = Bar('{}'.format(opt.exp_id), max=num_iters) time_stats = ['tot', 'load', 'pre', 'net', 'dec', 'post', 'merge'] avg_time_stats = {t: AverageMeter() for t in time_stats} for ind, (img_id, pre_processed_images) in enumerate(data_loader): # pre_processed_images['img_info'] = dataset.coco.loadImgs([img_id])[0] ret = detector.run(pre_processed_images) results[img_id] = ret['results'] Bar.suffix = '[{0}/{1}]|Tot: {total:} |ETA: {eta:} '.format( ind, num_iters, total=bar.elapsed_td, eta=bar.eta_td) # if opt.debug: # # img = pre_processed_images['image'].detach().cpu().numpy().squeeze() # print(type(ret['results'])) # vis_dets(ret, img_id, dataset, split=split) for t in avg_time_stats: avg_time_stats[t].update(ret[t]) Bar.suffix += '|{} {tm.val:.3f}s ({tm.avg:.3f}s) '.format( t, tm=avg_time_stats[t]) bar.next() bar.finish() sava_path = opt.save_dir + '/{}_results'.format(split) dataset.save_pkl(results, sava_path) dataset.save_dota_format(results, sava_path) if split != 'test': dataset.dota_eval(sava_path + '/Task1_{:s}.txt') mean_ap, eval_results = dataset.coco_eval(results) print(mean_ap, eval_results)