def eval(cfg): # Setup seeds torch.manual_seed(cfg.get("seed", 1337)) torch.cuda.manual_seed(cfg.get("seed", 1337)) np.random.seed(cfg.get("seed", 1337)) random.seed(cfg.get("seed", 1337)) # Setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Setup evaluation data loader_source = get_loader(cfg, "train") # data_eval_labels = utils.recursive_glob(os.path.join(cfg["data"]["path"], 'labels')) # Setup model model = Unet(cfg).to(device) checkpoint = torch.load(cfg["training"]["checkpoint"]) model.load_state_dict(checkpoint["model_state"]) stats = None model.eval() for images, labels in tqdm.tqdm(loader_source): model.set_input(images, labels) model.forward() if stats is None: stats = [StatsRecorder() for i in range(len(model.hooks))] for i, hook in enumerate(model.hooks): activation = hook.output b, c, h, w = activation.shape activation = activation.transpose(0, 1).reshape(c, -1).transpose(0, 1) stats[i].update(activation.cpu().data.numpy()) print([s.mean for s in stats]) print([s.std for s in stats])
if initial_checkpoint is not None: model, optimizer, epochnum, valid_loss_min = load_ckp(checkpoint_path+initial_checkpoint, model, optimizer) print(f'initial ckp: {epochnum}') i = i + epochnum for epoch in range(epochs): running_train_loss = [] running_train_score = [] model.train() for step,(image, mask) in enumerate(train_loader): image = image.cuda() mask = mask.cuda() pred_mask = model.forward(image) # forward propogation loss = criterion(pred_mask, mask) score = iou_batch(pred_mask, mask) optimizer.zero_grad() # setting gradient to zero loss.backward() optimizer.step() running_train_loss.append(loss.item()) running_train_score.append(score) print(f'batch DiceBCELoss: {loss.item()}') print(f'batch iou: {score}') else: ### do valid after epoch ### running_val_loss = [] running_val_score = [] with torch.no_grad():