def get_model_and_optimizer(device):
model = UNet(
in_channels=1,
out_classes=1,
dimensions=3,
normalization='Group',
num_encoding_blocks=3,
out_channels_first_layer=8,
upsampling_type='conv',
padding=2,
activation='PReLU',
dropout=0,
).to(device)
optimizer = torch.optim.AdamW(model.parameters())
return model, optimizer
def __init__(self, input_channels=1, num_classes=1, num_filters_=[32,64,128,192], latent_dim=6, beta=10.0, initializers=None, isotropic=False,
device='cuda'):
super(ProbabilisticUnet, self).__init__()
self.input_channels = input_channels
self.num_classes = num_classes
self.num_filters = num_filters_
self.latent_dim = latent_dim
self.no_convs_per_block = num_convs_per_block
self.no_convs_fcomb = num_convs_fcomb
self.initializers = initializers
self.beta = beta
self.z_prior_sample = 0
self.isotropic = isotropic
self.unet = UNet(self.input_channels, self.num_classes, num_filters=self.num_filters, if_last_layer=False).to(device)
self.prior = AxisAlignedGaussian(self.input_channels, self.num_filters, self.no_convs_per_block, self.latent_dim, self.initializers, isotropic=isotropic).to(device)
self.posterior = AxisAlignedGaussian(self.input_channels, self.num_filters, self.no_convs_per_block, self.latent_dim, self.initializers, isotropic=isotropic, posterior=True).to(device)
self.fcomb = Fcomb(self.num_filters, self.latent_dim, self.input_channels,
self.num_classes, self.no_convs_fcomb, self.initializers, use_tile=True, device=device).to(device)
def eval(data):
net = UNet(in_channels=1, n_classes=1, bilinear=True).to(device)
load_dict = torch.load(eval_model, map_location='cuda:0')
net.load_state_dict(load_dict['state_dict'])
# net.load_state_dict(load_dict)
net.eval()
with torch.no_grad():
with tqdm(total=len(data.test_indices), unit='patch') as pbar:
for step, (patch, mask, _) in enumerate(data.test_loader):
patch = patch.to(device)
mask = mask.to(device)
recon = net(patch)
imageio.imwrite(
os.path.join(recon_dir,
str(step) + '_image.png'),
patch[0].cpu().numpy().T)
imageio.imwrite(
os.path.join(recon_dir,
str(step) + '_mask.png'),
mask[0].cpu().numpy().T)
imageio.imwrite(
os.path.join(recon_dir,
str(step) + '_recon.png'),
-recon[0].cpu().numpy().T.astype(np.uint8))
pbar.update(data.batch_size)
mask[0].cpu().numpy().T)
imageio.imwrite(
os.path.join(recon_dir,
str(step) + '_recon.png'),
-recon[0].cpu().numpy().T.astype(np.uint8))
pbar.update(data.batch_size)
def save_checkpoint(state, save_path, filename):
filename = os.path.join(save_path, filename)
torch.save(state, filename)
if __name__ == '__main__':
dataset = LIDC_IDRI(dataset_location=data_dir,
joint_transform=None,
input_transform=None,
target_transform=target_transfm,
random=random)
dataloader = Dataloader(dataset,
batch_size=batch_size,
small=partial_data,
shuffle_indices=shuffle_indices)
net = UNet(in_channels=1,
n_classes=1,
bilinear=True,
num_filters=num_filters).to(device)
if not random:
print('always using first experts annotation')
train(dataloader, net)
image_width, image_height = 256, 256
num_ecpochs = 1
batch_size = 1
# Tensorboard callback and logging directory
logdir = "logs/scalars/" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir,
update_freq='batch')
# Training dataset
dataset_train = ShapesDataset()
dataset_train.load_shapes(train_num_samples, image_height, image_width)
dataset_train.prepare()
# Create U-Network
unet = UNet()
unet.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=["accuracy"])
# Organize the dataset in two numpy arrays
image = [
dataset_train.load_image(image_id).astype(np.float32) / 255
for image_id in dataset_train.image_ids
]
masks = []
for image_id in dataset_train.image_ids:
mask, classes = dataset_train.load_mask(image_id)
# Treshold the segmentation images to get b&w alpha mask
mask_treshold = np.expand_dims(np.where(mask[:, :, 0] > 0, 255,
parser.add_argument("--result_folder", type=str, required=True)
parser.add_argument("--device", type=str, default='cpu')
parser.add_argument("--min_object_size", type=float, default=0)
parser.add_argument("--max_object_size", type=float, default=np.inf)
parser.add_argument("--dpi", type=float, default=False)
parser.add_argument("--dpm", type=float, default=False)
parser.add_argument("--visualize", type=bool, default=False)
args = parser.parse_args()
# determining dpm
dpm = args.dpm if not args.dpi else dpi_to_dpm(args.dpi)
print("Loading dataset...")
predict_dataset = ReadTestDataset(args.images_path)
device = torch.device(args.device)
print("Dataset loaded")
print("Loading model...")
unet = UNet(3, 3)
unet.load_state_dict(torch.load(args.model, map_location=device))
model = ModelWrapper(unet, args.result_folder, cuda_device=device)
print("Model loaded")
print("Measuring images...")
model.measure_large_images(predict_dataset,
export_path=args.result_folder,
visualize_bboxes=args.visualize,
filter=[args.min_object_size, args.max_object_size],
dpm=dpm,
verbose=True,
tile_res=(256, 256))
class ProbabilisticUnet(nn.Module):
"""
A probabilistic UNet (https://arxiv.org/abs/1806.05034) implementation.
input_channels: the number of channels in the image (1 for greyscale and 3 for RGB)
num_classes: the number of classes to predict
num_filters: is a list consisint of the amount of filters layer
latent_dim: dimension of the latent space
no_cons_per_block: no convs per block in the (convolutional) encoder of prior and posterior
"""
def __init__(self, input_channels=1, num_classes=1, num_filters_=[32,64,128,192], latent_dim=6, beta=10.0, initializers=None, isotropic=False,
device='cuda'):
super(ProbabilisticUnet, self).__init__()
self.input_channels = input_channels
self.num_classes = num_classes
self.num_filters = num_filters_
self.latent_dim = latent_dim
self.no_convs_per_block = num_convs_per_block
self.no_convs_fcomb = num_convs_fcomb
self.initializers = initializers
self.beta = beta
self.z_prior_sample = 0
self.isotropic = isotropic
self.unet = UNet(self.input_channels, self.num_classes, num_filters=self.num_filters, if_last_layer=False).to(device)
self.prior = AxisAlignedGaussian(self.input_channels, self.num_filters, self.no_convs_per_block, self.latent_dim, self.initializers, isotropic=isotropic).to(device)
self.posterior = AxisAlignedGaussian(self.input_channels, self.num_filters, self.no_convs_per_block, self.latent_dim, self.initializers, isotropic=isotropic, posterior=True).to(device)
self.fcomb = Fcomb(self.num_filters, self.latent_dim, self.input_channels,
self.num_classes, self.no_convs_fcomb, self.initializers, use_tile=True, device=device).to(device)
def forward(self, patch, segm, training=False):
"""
Construct prior latent space for patch and run patch through UNet,
in case training is True also construct posterior latent space
"""
if training:
self.posterior_z = self.posterior.forward(patch, segm)
self.prior_z = self.prior.forward(patch)
self.unet_features = self.unet.forward(patch)
def sample_(self, testing=False):
"""
Sample a segmentation by reconstructing from a prior sample
and combining this with UNet features
"""
if testing == False:
z_prior = self.prior_z.rsample()
self.z_prior_sample = z_prior
else:
#You can choose whether you mean a sample or the mean here. For the GED it is important to take a sample.
#z_prior = self.prior_z.base_dist.loc
z_prior = self.prior_z.sample()
self.z_prior_sample = z_prior
return self.fcomb.forward(self.unet_features, self.z_prior_sample)
def reconstruct(self, use_posterior_mean=False, calculate_posterior=False, z_posterior=None):
"""
Reconstruct a segmentation from a posterior sample (decoding a posterior sample) and UNet feature map
use_posterior_mean: use posterior_mean instead of sampling z_q
calculate_posterior: use a provided sample or sample from posterior latent space
"""
assert use_posterior_mean == use_posterior_mean
if use_posterior_mean:
z_posterior = self.posterior_z.loc
else:
if calculate_posterior:
z_posterior = self.posterior_z.rsample()
return self.fcomb.forward(self.unet_features, z_posterior)
def elbo(self, segm, reconstruct_posterior_mean=False, patch=None):
assert type(patch) != None
"""
Calculate the evidence lower bound of the log-likelihood of P(Y|X)
"""
criterion = nn.BCEWithLogitsLoss(reduction='none')
self.kl = torch.mean(kl.kl_divergence(self.posterior_z, self.prior_z))
#Here we use the posterior sample sampled above
self.reconstruction = self.reconstruct(use_posterior_mean=reconstruct_posterior_mean, calculate_posterior=True)
reconstruction_loss = criterion(input=self.reconstruction, target=segm)
self.reconstruction_loss = torch.sum(reconstruction_loss)
self.mean_reconstruction_loss = torch.mean(reconstruction_loss)
return self.reconstruction_loss + self.beta * self.kl
def output_predict_manifold(self, patch=None):
assert type(patch) != None
assert self.latent_dim == 2
nx = ny = 10
x_values = np.linspace(.05, .95, nx)
y_values = np.linspace(.05, .95, ny)
canvas = np.empty((128 * ny, 128 * nx))
for i, yi in enumerate(x_values):
for j, xi in enumerate(y_values):
z_posterior = np.array([norm.ppf(xi), norm.ppf(yi)]).astype('float32').T
z_posterior = torch.unsqueeze(torch.tensor(z_posterior), dim=0).cuda()
canvas[(nx-i-1)*128:(nx-i)*128, j*128:(j+1)*128] = self.reconstruct(z_posterior=z_posterior).cpu().numpy().reshape(128, 128)
return canvas
def output_predict_tensor(self, num_sample=10, patch=None):
assert type(patch) != None
r = []
for samp in range(num_sample):
# z_posterior = self.prior.sample_()
reconstruction = self.sample_(testing=True).cpu().numpy()
r.append(reconstruction)
return r
# Validation dataset
dataset_val = ShapesDataset()
dataset_val.load_shapes(eval_num_samples, image_height, image_width)
dataset_val.prepare()
# Organize the evaluation set in two numpy arrays
image = [
dataset_val.load_image(image_id).astype(np.float16)
for image_id in dataset_val.image_ids
]
masks = []
for image_id in dataset_val.image_ids:
mask, classes = dataset_val.load_mask(image_id)
# Treshold the segmentation images to get b&w alpha mask
mask_treshold = np.expand_dims(np.where(mask[:, :, 0] > 0, 255,
0).astype(np.float16),
axis=3)
masks.append(mask_treshold)
# Create U-Network and load weights
unet = UNet()
unet.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=["accuracy"])
unet.load_weights("./weights/tf_unet_toy_network")
loss, acc = unet.evaluate(np.array(image), np.array(masks), verbose=2)
print("Restored model and ran evaluation, accuracy: {:5.2f}%".format(100 *
acc))
tf_validate = make_transform(crop=(512, 512),
long_mask=True,
rotate_range=False,
p_flip=0.0,
normalize=False,
color_jitter_params=None)
# creating checkpoint folder
model_name = args.model_name
file_dir = os.path.split(os.path.realpath(__file__))[0]
results_folder = os.path.join(file_dir, '..', 'checkpoints', model_name)
if not os.path.exists(results_folder):
os.makedirs(results_folder)
# load model
unet = UNet(3, 3)
if args.trained_model_path is not None:
unet.load_state_dict(torch.load(args.trained_model_path))
loss = SoftDiceLoss(weight=torch.Tensor([1, 5, 5]))
optimizer = optim.Adam(unet.parameters(), lr=args.initial_lr)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
patience=50,
verbose=True)
cuda_device = torch.device(args.device)
model = ModelWrapper(unet,
loss=loss,
optimizer=optimizer,
scheduler=scheduler,
results_folder=results_folder,