learning_rate_scheduler=scheduler) else: print("Making directories") os.makedirs(args.folder, exist_ok=True) start_epoch, val_metric_per_epoch = 0, [] shutil.copy2(args.params, args.folder) param_shapes = print_para(model) num_batches = 0 for epoch_num in range(start_epoch, params['trainer']['num_epochs'] + start_epoch): train_results = [] norms = [] model.train() for b, (time_per_batch, batch) in enumerate( time_batch(train_loader if args.no_tqdm else tqdm(train_loader), reset_every=ARGS_RESET_EVERY)): batch = _to_gpu(batch) optimizer.zero_grad() output_dict = model(**batch) loss = output_dict['loss'].mean( ) + output_dict['cnn_regularization_loss'].mean() loss.backward() num_batches += 1 if scheduler: scheduler.step_batch(num_batches) norms.append( clip_grad_norm(model.named_parameters(), max_norm=params['trainer']['grad_norm'], clip=True,
params = Params.from_file(args.params) print("Loading {} for {}".format(params['model'].get('type', 'WTF?'), mode), flush=True) model = Model.from_params(vocab=vcr_dataset.vocab, params=params['model']) for submodule in model.detector.backbone.modules(): if isinstance(submodule, BatchNorm2d): submodule.track_running_stats = False model_state = torch.load(getattr(args, f'{mode}_ckpt'), map_location=device_mapping(-1)) model.load_state_dict(model_state) model = DataParallel(model).cuda() if NUM_GPUS > 1 else model.cuda() model.eval() test_probs = [] test_ids = [] for b, (time_per_batch, batch) in enumerate(time_batch(test_loader)): with torch.no_grad(): batch = _to_gpu(batch) output_dict = model(**batch) test_probs.append(output_dict['label_probs'].detach().cpu().numpy()) test_ids += [m['annot_id'] for m in batch['metadata']] if (b > 0) and (b % 10 == 0): print("Completed {}/{} batches in {:.3f}s".format(b, len(test_loader), time_per_batch * 10), flush=True) probs_grp.append(np.concatenate(test_probs, 0)) ids_grp.append(test_ids) ################################################################################ # This is the part you'll care about if you want to submit to the leaderboard! ################################################################################