def get_isbi_loader(config):
"""
Gets ISBI2012 Loader given a the path to a configuration file. Supported file formats are
HDF5 (.h5) and TIF (.tif or .tiff).
Parameters
----------
config : str or dict
(Path to) Data configuration.
Returns
-------
torch.utils.data.dataloader.DataLoader
Data loader built as configured.
"""
config = yaml2dict(config)
dataset = ISBI2012Dataset.from_config(config)
loader = DataLoader(dataset, **config.get('loader_config'))
return loader
def load_Unet3D(config_dict):
"""
loads Unet3D with from neurofire
:param config_dict: dictionary with all important paths
:return: Unet3D model
"""
import torch
import neurofire.models as models
from inferno.utils.io_utils import yaml2dict
config = yaml2dict(config_dict["train_config_folder"])
model_name = config.get('model_name')
model = getattr(models, model_name)(**config.get('model_kwargs'))
if torch.cuda.is_available():
model.cuda()
return model
def get_cremi_loader(config):
"""
Get Cremi loader given a the path to a configuration file.
Parameters
----------
config : str or dict
(Path to) Data configuration.
Returns
-------
torch.utils.data.dataloader.DataLoader
Data loader built as configured.
"""
config = yaml2dict(config)
loader_config = config.pop('loader_config')
datasets = CremiDatasets.from_config(config)
loader = DataLoader(datasets, **loader_config)
return loader
def make_validation_config(template_config,
validation_config_file,
affinity_config=None):
template = os.path.join(template_config, 'validation_config.yml')
template = yaml2dict(template)
key = 'nucleus_instance_labels'
affinity_config.update({
'segmentation_to_binary': False,
'retain_segmentation': True
})
template['volume_config']['segmentation'][
'affinity_config'] = affinity_config
template['volume_config']['segmentation']['dtype'] = 'int64'
key = 'volumes/labels/' + key
for block_id in (2, 4):
template['volume_config']['segmentation']['path_in_file'][str(
block_id)] = key
with open(validation_config_file, 'w') as f:
yaml.dump(template, f)
def get_criterion_and_optimizer(net, config_dict):
"""
Initializes criterion and optimizer for net
:param net: NeuralNet
:param config_dict: dictionary with all important paths
:return: criterion and optimizer
"""
from inferno.utils.io_utils import yaml2dict
import torch.optim as optim
from SorensenDiceLoss import SorensenDiceLoss
criterion = SorensenDiceLoss()
config = yaml2dict(config_dict["train_config_folder"])
optimizer_kwargs = config.get('training_optimizer_kwargs')
optimizer = optim.SGD(net.parameters(),
lr=optimizer_kwargs.get('lr'),
weight_decay=optimizer_kwargs.get('weight_decay'))
return criterion, optimizer
def __init__(self,
path,
defect_augmentation_config,
name=None,
path_in_file=None,
data_slice=None,
dtype='float32',
ignore_slice_list=None,
mean=None,
std=None,
sigma=None,
zero_mean_unit_variance=True,
p_augment_ws=0.,
**slicing_config):
super().__init__(path=path,
path_in_file=path_in_file,
data_slice=data_slice,
name=name,
dtype=dtype,
mean=mean,
std=std,
sigma=sigma,
p_augment_ws=p_augment_ws,
zero_mean_unit_variance=zero_mean_unit_variance,
**slicing_config)
defect_augmentation_config = yaml2dict(defect_augmentation_config)
defect_augmentation_config.update(
{'ignore_slice_list': ignore_slice_list})
self.defect_augmentation = DefectAugmentation.from_config(
defect_augmentation_config)
self.cast = Cast(self.dtype)
# Check if we should keep track of defected slices:
self.keep_track_defected_slices = len(
self.defect_augmentation.keep_track_of) > 0
def make_validation_config(validation_config_file, offsets):
template = yaml2dict('./template_config/validation_config_hed.yml')
template['volume_config']['segmentation']['affinity_offsets'] = offsets
with open(validation_config_file, 'w') as f:
yaml.dump(template, f)
def make_validation_config(config_folder, validation_config_file):
template = yaml2dict(f'./{config_folder}/validation_config.yml')
with open(validation_config_file, 'w') as f:
yaml.dump(template, f)
def make_data_config(config_folder, data_config_file, n_batches):
template = yaml2dict(f'./{config_folder}/data_config.yml')
template['loader_config']['batch_size'] = n_batches
template['loader_config']['num_workers'] = 8 * n_batches
with open(data_config_file, 'w') as f:
yaml.dump(template, f)
def get_autophagosom_loader(config):
config = yaml2dict(config)
loader_config = config.pop('loader_config')
datasets = AutophagosomDatasets.from_config(config)
loader = DataLoader(datasets, **loader_config)
return loader
LOG_DIRECTORY = ensure_dir('./logs_2')
BATCHSIZE = 8
N_DIRECTIONS = 8
# unsq = transforms.Lambda(lambda x: torch.unsqueeze(x, 0))
transpose = transforms.Lambda(lambda x: torch.transpose(x, 0, 1))
squeeze = transforms.Lambda(lambda x: torch.squeeze(x, 1))
fromnumpy = transforms.Lambda(lambda x: torch.from_numpy(x))
trans = transforms.Compose([fromnumpy])
trans2 = transforms.Compose([fromnumpy, squeeze])
imageset_train = HDF5VolumeLoader(path='./train-volume.h5', path_in_h5_dataset='data',
transforms=trans, **yaml2dict('config_train.yml')['slicing_config'])
labelset_train = HDF5VolumeLoader(path='./stardistance.h5', path_in_h5_dataset='data',
transforms=trans2, **yaml2dict('config_train.yml')['slicing_config_truth'])
trainset = Zip(imageset_train, labelset_train)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCHSIZE,
shuffle=True, num_workers=2)
imageset_val = HDF5VolumeLoader(path='./val-volume.h5', path_in_h5_dataset='data',
transforms=trans, **yaml2dict('config_val.yml')['slicing_config'])
labelset_val = HDF5VolumeLoader(path='./stardistance_val.h5', path_in_h5_dataset='data',
transforms=trans2, **yaml2dict('config_val.yml')['slicing_config_truth'])
trainset = Zip(imageset_val, labelset_val)
valloader = torch.utils.data.DataLoader(trainset, batch_size=BATCHSIZE,
shuffle=True, num_workers=2)
net = torch.nn.Sequential(
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
transpose = transforms.Lambda(lambda x: torch.transpose(x, 0, 1))
squeeze = transforms.Lambda(lambda x: torch.squeeze(x, 1))
fromnumpy = transforms.Lambda(lambda x: torch.from_numpy(x))
trans = transforms.Compose([fromnumpy])
trans2 = transforms.Compose([fromnumpy, squeeze])
imageset_val = HDF5VolumeLoader(
path='./val-volume.h5',
path_in_h5_dataset='data',
transforms=trans,
**yaml2dict('config_val.yml')['slicing_config'])
labelset_val = HDF5VolumeLoader(
path='./stardistance_val.h5',
path_in_h5_dataset='data',
transforms=trans2,
**yaml2dict('config_val.yml')['slicing_config_truth'])
trainer = Trainer()
trainer.load(from_directory='checkpoints', map_location='cpu', best=False)
result, loss = trainer.apply_model_and_loss(
imageset_val[5].unsqueeze(0).unsqueeze(0), #.to('cuda'),
labelset_val[5].unsqueeze(0).unsqueeze(0)) #.to('cuda'))
print(loss)
def make_train_config(train_config_file, gpus):
template = yaml2dict('./template_config/train_config.yml')
template['devices'] = gpus
with open(train_config_file, 'w') as f:
yaml.dump(template, f)
def from_config(cls, config):
config = yaml2dict(config)
volume_config = config.get('volume_config')
slicing_config = config.get('slicing_config')
return cls(volume_config=volume_config, slicing_config=slicing_config)
def make_validation_config(validation_config_file, affinity_config):
template = yaml2dict('./template_config/validation_config.yml')
template['volume_config']['segmentation'][
'affinity_config'] = affinity_config
with open(validation_config_file, 'w') as f:
yaml.dump(template, f)
def make_validation_config(validation_config_file, name):
template = yaml2dict('./template_config/validation_%s.yaml' % name)
template['master_config']['affinity_config'] = None
template['master_config']['train_semantic'] = True
with open(validation_config_file, 'w') as f:
yaml.dump(template, f)
def make_train_config(train_config_file, offsets, gpus):
template = yaml2dict('./template_config/train_config.yml')
template['model_kwargs']['out_channels'] = len(offsets)
template['devices'] = gpus
with open(train_config_file, 'w') as f:
yaml.dump(template, f)
def load_volume(inference_config):
config = yaml2dict(inference_config)
vol_config = config['volume_config']['raw']
slicing_config = config['slicing_config']
return RawVolumeHDF5(**vol_config, **slicing_config)
class LabelToTarget(Transform):
def __init__(self):
super().__init__()
def batch_function(self, tensors):
prediction, target = tensors
target = np.moveaxis(star_dist(target[0].numpy(), N_DIRECTIONS, opencl=OPENCL_AVAILABLE), -1, 0)
return prediction, target
tosignedint = transforms.Lambda(lambda x: torch.tensor(np.int32(x), dtype=torch.int32))
sdist = transforms.Lambda(lambda x: np.moveaxis(star_dist(x[0], N_DIRECTIONS, opencl=OPENCL_AVAILABLE), -1, 0))
labeltotarget = transforms.Compose([sdist])
train_images = HDF5VolumeLoader(path='./train-volume.h5', path_in_h5_dataset='data',
**yaml2dict('config_train.yml')['slicing_config'])
train_labels = HDF5VolumeLoader(path='labeled_segmentation.h5', path_in_h5_dataset='data',
transforms=tosignedint, **yaml2dict('config_train.yml')['slicing_config'])
trainset = Zip(train_images, train_labels)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCHSIZE,
shuffle=True, num_workers=2)
val_images = HDF5VolumeLoader(path='./val-volume.h5', path_in_h5_dataset='data',
**yaml2dict('config_val.yml')['slicing_config'])
val_labels = HDF5VolumeLoader(path='labeled_segmentation_validation.h5', path_in_h5_dataset='data',
transforms=tosignedint, **yaml2dict('config_val.yml')['slicing_config'])
valset = Zip(val_images, val_labels)
valloader = torch.utils.data.DataLoader(trainset, batch_size=BATCHSIZE,
shuffle=True, num_workers=2)
