def train(self, channels=None, **config_args):
#limit_gpu_memory(fraction=1)
if channels is None:
channels = self.train_channels
for ch in channels:
print("-- Training channel {}...".format(ch))
(X, Y), (X_val, Y_val), axes = load_training_data(
self.get_training_patch_path() /
'CH_{}_training_patches.npz'.format(ch),
validation_split=0.1,
verbose=False)
c = axes_dict(axes)['C']
n_channel_in, n_channel_out = X.shape[c], Y.shape[c]
config = Config(axes,
n_channel_in,
n_channel_out,
train_epochs=self.train_epochs,
train_steps_per_epoch=self.train_steps_per_epoch,
train_batch_size=self.train_batch_size,
**config_args)
# Training
model = CARE(config,
'CH_{}_model'.format(ch),
basedir=pathlib.Path(self.out_dir) / 'models')
# Show learning curve and example validation results
try:
history = model.train(X, Y, validation_data=(X_val, Y_val))
except tf.errors.ResourceExhaustedError:
print(
"ResourceExhaustedError: Aborting...\n Training data too big for GPU. Are other GPU jobs running? Perhaps, reduce batch-size or patch-size?"
)
return
#print(sorted(list(history.history.keys())))
plt.figure(figsize=(16, 5))
plot_history(history, ['loss', 'val_loss'],
['mse', 'val_mse', 'mae', 'val_mae'])
plt.figure(figsize=(12, 7))
_P = model.keras_model.predict(X_val[:5])
plot_some(X_val[:5], Y_val[:5], _P, pmax=99.5, cmap="gray")
plt.suptitle('5 example validation patches\n'
'top row: input (source), '
'middle row: target (ground truth), '
'bottom row: predicted from source')
plt.show()
print("-- Export model for use in Fiji...")
model.export_TF()
print("Done")
def dev(args):
import json
# Load and parse training data
(X,
Y), (X_val,
Y_val), axes = load_training_data(args.train_data,
validation_split=args.valid_split,
axes=args.axes,
verbose=True)
c = axes_dict(axes)['C']
n_channel_in, n_channel_out = X.shape[c], Y.shape[c]
# Model config
print('args.resume: ', args.resume)
if args.resume:
# If resuming, config=None will reload the saved config
config = None
print('Attempting to resume')
elif args.config:
print('loading config from args')
config_args = json.load(open(args.config))
config = Config(**config_args)
else:
config = Config(axes,
n_channel_in,
n_channel_out,
probabilistic=args.prob,
train_steps_per_epoch=args.steps,
train_epochs=args.epochs)
print(vars(config))
# Load or init model
model = CARE(config, args.model_name, basedir='models')
# Training, tensorboard available
history = model.train(X, Y, validation_data=(X_val, Y_val))
# Plot training results
print(sorted(list(history.history.keys())))
plt.figure(figsize=(16, 5))
plot_history(history, ['loss', 'val_loss'],
['mse', 'val_mse', 'mae', 'val_mae'])
plt.savefig(args.model_name + '_training.png')
# Export model to be used w/ csbdeep fiji plugins and KNIME flows
model.export_TF()
n_channel_in,
n_channel_out,
unet_n_depth=4,
train_epochs=100,
train_steps_per_epoch=400,
train_batch_size=16,
train_reduce_lr={
'patience': 5,
'factor': 0.5
})
print(config)
vars(config)
# In[5]:
model = CARE(config=config, name=ModelName, basedir=BaseDir)
# In[6]:
history = model.train(X, Y, validation_data=(X_val, Y_val))
# In[7]:
# In[8]:
# In[9]:
model.export_TF()
# In[10]:
def train(X, Y, X_val, Y_val, axes, model_name, csv_file,probabilistic, validation_split, patch_size, **kwargs):
"""Trains CARE model with patches previously created.
CARE model parameters configurated via 'Config' object:
* parameters of the underlying neural network,
* the learning rate,
* the number of parameter updates per epoch,
* the loss function, and
* whether the model is probabilistic or not.
Parameters
----------
X : np.array
Input data X for training
Y : np.array
Ground truth data Y for training
X_val : np.array
Input data X for validation
Y_val : np.array
Ground truth Y for validation
axes : str
Semantic order of the axis in the image
train_steps_per_epoch : int
Number of training steps per epochs
train_epochs : int
Number of training epochs
model_name : str
Name of the model to be saved after training
Returns
-------
history
Object with the loss values saved
"""
config = Config(axes, n_channel_in=1, n_channel_out=1, probabilistic=probabilistic, allow_new_parameters=True, **kwargs)
model = CARE(config, model_name, basedir='models')
# # Training
# [TensorBoard](https://www.tensorflow.org/programmers_guide/summaries_and_tensorboard) from the current working directory with `tensorboard --logdir=.`
# Then connect to [http://localhost:6006/](http://localhost:6006/) with your browser.
history = model.train(X, Y, validation_data=(X_val, Y_val))
# df = pd.DataFrame()
# df['probabilistic'] = [config.probabilistic]
# df['batch_size'] = config.train_batch_size
# df['train_epochs'] = config.train_epochs
# df['lr'] = config.train_learning_rate
# df['train_steps_per_epoch'] = config.train_steps_per_epoch
# df['unet_last_activation'] = config.unet_last_activation
# df['unet_n_depth'] = config.unet_n_depth
# df['unet_n_first'] = config.unet_n_first
# df['unet_residual'] = config.unet_residual
# df['patch_size'] = str(patch_size)
# df['validation_split'] = validation_split
# last_value = len(history.history['loss']) - 1
# #dict = kwargs
# #data = dict.update({'loss':history.history['loss'][last_value], })
# df['val_loss'] = history.history['val_loss'][last_value]
# df['loss'] = history.history['loss'][last_value]
# df['mse']=history.history['mse'][last_value]
# df['mae']=history.history['mae'][last_value]
# df['val_mse']=history.history['val_mse'][last_value]
# df['val_mae']=history.history['val_mae'][last_value]
# #df = pd.DataFrame.from_dict(data)
# #print('File saved in:', os.getcwd()+'/'+csv_file)
# df.to_csv('/data/'+csv_file)
model.export_TF()
return history
def training(self):
#Loading Files and Plot examples
basepath = 'data/'
training_original_dir = 'training/original/'
training_ground_truth_dir = 'training/ground_truth/'
validation_original_dir = 'validation/original/'
validation_ground_truth_dir = 'validation/ground_truth/'
import glob
from skimage import io
from matplotlib import pyplot as plt
training_original_files = sorted(
glob.glob(basepath + training_original_dir + '*.tif'))
training_original_file = io.imread(training_original_files[0])
training_ground_truth_files = sorted(
glob.glob(basepath + training_ground_truth_dir + '*.tif'))
training_ground_truth_file = io.imread(training_ground_truth_files[0])
print("Training dataset's number of files and dimensions: ",
len(training_original_files), training_original_file.shape,
len(training_ground_truth_files),
training_ground_truth_file.shape)
training_size = len(training_original_file)
validation_original_files = sorted(
glob.glob(basepath + validation_original_dir + '*.tif'))
validation_original_file = io.imread(validation_original_files[0])
validation_ground_truth_files = sorted(
glob.glob(basepath + validation_ground_truth_dir + '*.tif'))
validation_ground_truth_file = io.imread(
validation_ground_truth_files[0])
print("Validation dataset's number of files and dimensions: ",
len(validation_original_files), validation_original_file.shape,
len(validation_ground_truth_files),
validation_ground_truth_file.shape)
validation_size = len(validation_original_file)
if training_size == validation_size:
size = training_size
else:
print(
'Training and validation images should be of the same dimensions!'
)
plt.figure(figsize=(16, 4))
plt.subplot(141)
plt.imshow(training_original_file)
plt.subplot(142)
plt.imshow(training_ground_truth_file)
plt.subplot(143)
plt.imshow(validation_original_file)
plt.subplot(144)
plt.imshow(validation_ground_truth_file)
#preparing inputs for NN from pairs of 32bit TIFF image files with intensities in range 0..1. . Run it only once for each new dataset
from csbdeep.data import RawData, create_patches, no_background_patches
training_data = RawData.from_folder(
basepath=basepath,
source_dirs=[training_original_dir],
target_dir=training_ground_truth_dir,
axes='YX',
)
validation_data = RawData.from_folder(
basepath=basepath,
source_dirs=[validation_original_dir],
target_dir=validation_ground_truth_dir,
axes='YX',
)
# pathces will be created further in "data augmentation" step,
# that's why patch size here is the dimensions of images and number of pathes per image is 1
size1 = 64
X, Y, XY_axes = create_patches(
raw_data=training_data,
patch_size=(size1, size1),
patch_filter=no_background_patches(0),
n_patches_per_image=1,
save_file=basepath + 'training.npz',
)
X_val, Y_val, XY_axes = create_patches(
raw_data=validation_data,
patch_size=(size1, size1),
patch_filter=no_background_patches(0),
n_patches_per_image=1,
save_file=basepath + 'validation.npz',
)
#Loading training and validation data into memory
from csbdeep.io import load_training_data
(X, Y), _, axes = load_training_data(basepath + 'training.npz',
verbose=False)
(X_val,
Y_val), _, axes = load_training_data(basepath + 'validation.npz',
verbose=False)
X.shape, Y.shape, X_val.shape, Y_val.shape
from csbdeep.utils import axes_dict
c = axes_dict(axes)['C']
n_channel_in, n_channel_out = X.shape[c], Y.shape[c]
batch = len(
X
) # You should define number of batches according to the available memory
#batch=1
seed = 1
from keras.preprocessing.image import ImageDataGenerator
data_gen_args = dict(samplewise_center=False,
samplewise_std_normalization=False,
zca_whitening=False,
fill_mode='reflect',
rotation_range=30,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True
#width_shift_range=0.2,
#height_shift_range=0.2,
)
# training
image_datagen = ImageDataGenerator(**data_gen_args)
mask_datagen = ImageDataGenerator(**data_gen_args)
image_datagen.fit(X, augment=True, seed=seed)
mask_datagen.fit(Y, augment=True, seed=seed)
image_generator = image_datagen.flow(X, batch_size=batch, seed=seed)
mask_generator = mask_datagen.flow(Y, batch_size=batch, seed=seed)
generator = zip(image_generator, mask_generator)
# validation
image_datagen_val = ImageDataGenerator(**data_gen_args)
mask_datagen_val = ImageDataGenerator(**data_gen_args)
image_datagen_val.fit(X_val, augment=True, seed=seed)
mask_datagen_val.fit(Y_val, augment=True, seed=seed)
image_generator_val = image_datagen_val.flow(X_val,
batch_size=batch,
seed=seed)
mask_generator_val = mask_datagen_val.flow(Y_val,
batch_size=batch,
seed=seed)
generator_val = zip(image_generator_val, mask_generator_val)
# plot examples
x, y = generator.__next__()
x_val, y_val = generator_val.__next__()
plt.figure(figsize=(16, 4))
plt.subplot(141)
plt.imshow(x[0, :, :, 0])
plt.subplot(142)
plt.imshow(y[0, :, :, 0])
plt.subplot(143)
plt.imshow(x_val[0, :, :, 0])
plt.subplot(144)
plt.imshow(y_val[0, :, :, 0])
import os
blocks = 2
channels = 16
learning_rate = 0.0004
learning_rate_decay_factor = 0.95
epoch_size_multiplicator = 20 # basically, how often do we decrease learning rate
epochs = 10
#comment='_side' # adds to model_name
import datetime
#model path
model_path = f'models/CSBDeep/model.h5'
if os.path.isfile(model_path):
print('Your model will be overwritten in the next cell')
kernel_size = 3
from csbdeep.models import Config, CARE
from keras import backend as K
best_mae = 1
steps_per_epoch = len(X) * epoch_size_multiplicator
validation_steps = len(X_val) * epoch_size_multiplicator
if 'model' in globals():
del model
if os.path.isfile(model_path):
os.remove(model_path)
for i in range(epochs):
print('Epoch:', i + 1)
learning_rate = learning_rate * learning_rate_decay_factor
config = Config(axes,
n_channel_in,
n_channel_out,
unet_kern_size=kernel_size,
train_learning_rate=learning_rate,
unet_n_depth=blocks,
unet_n_first=channels)
model = CARE(config, '.', basedir='models')
#os.remove('models/config.json')
if i > 0:
model.keras_model.load_weights(model_path)
model.prepare_for_training()
history = model.keras_model.fit_generator(
generator,
validation_data=generator_val,
validation_steps=validation_steps,
epochs=1,
verbose=0,
shuffle=True,
steps_per_epoch=steps_per_epoch)
if history.history['val_mae'][0] < best_mae:
best_mae = history.history['val_mae'][0]
if not os.path.exists('models/'):
os.makedirs('models/')
model.keras_model.save(model_path)
print(f'Validation MAE:{best_mae:.3E}')
del model
K.clear_session()
#model path
model_path = 'models/CSBDeep/model.h5'
config = Config(axes,
n_channel_in,
n_channel_out,
unet_kern_size=kernel_size,
train_learning_rate=learning_rate,
unet_n_depth=blocks,
unet_n_first=channels)
model = CARE(config, '.', basedir='models')
model.keras_model.load_weights(model_path)
model.export_TF()
def train(self, channels=None, **config_args):
# limit_gpu_memory(fraction=1)
if channels is None:
channels = self.train_channels
with Timer("Training"):
for ch in channels:
print("-- Training channel {}...".format(ch))
(X, Y), (X_val, Y_val), axes = load_training_data(
self.get_training_patch_path() /
"CH_{}_training_patches.npz".format(ch),
validation_split=0.1,
verbose=False,
)
c = axes_dict(axes)["C"]
n_channel_in, n_channel_out = X.shape[c], Y.shape[c]
config = Config(
axes,
n_channel_in,
n_channel_out,
train_epochs=self.train_epochs,
train_steps_per_epoch=self.train_steps_per_epoch,
train_batch_size=self.train_batch_size,
probabilistic=self.probabilistic,
**config_args,
)
# Training
# if (
# pathlib.Path(self.out_dir) / "models" / "CH_{}_model".format(ch)
# ).exists():
# print("config there already")
# config = None
model = CARE(
config,
"CH_{}_model".format(ch),
basedir=pathlib.Path(self.out_dir) / "models",
)
# Show learning curve and example validation results
try:
history = model.train(X, Y, validation_data=(X_val, Y_val))
except tf.errors.ResourceExhaustedError:
print(
" >> ResourceExhaustedError: Aborting...\n Training data too big for GPU. Are other GPU jobs running? Perhaps, reduce batch-size or patch-size?"
)
return
except tf.errors.UnknownError:
print(
" >> UnknownError: Aborting...\n No enough memory available on GPU... are other GPU jobs running?"
)
return
# print(sorted(list(history.history.keys())))
plt.figure(figsize=(16, 5))
plot_history(history, ["loss", "val_loss"],
["mse", "val_mse", "mae", "val_mae"])
plt.figure(figsize=(12, 7))
_P = model.keras_model.predict(X_val[:5])
if self.probabilistic:
_P = _P[..., 0]
plot_some(X_val[:5], Y_val[:5], _P, pmax=99.5, cmap="gray")
plt.suptitle("5 example validation patches\n"
"top row: input (source), "
"middle row: target (ground truth), "
"bottom row: predicted from source")
plt.show()
print("-- Export model for use in Fiji...")
model.export_TF()
print("Done")