def train(self, channels=None, **config_args): #limit_gpu_memory(fraction=1) if channels is None: channels = self.train_channels for ch in channels: print("-- Training channel {}...".format(ch)) (X, Y), (X_val, Y_val), axes = load_training_data( self.get_training_patch_path() / 'CH_{}_training_patches.npz'.format(ch), validation_split=0.1, verbose=False) c = axes_dict(axes)['C'] n_channel_in, n_channel_out = X.shape[c], Y.shape[c] config = Config(axes, n_channel_in, n_channel_out, train_epochs=self.train_epochs, train_steps_per_epoch=self.train_steps_per_epoch, train_batch_size=self.train_batch_size, **config_args) # Training model = CARE(config, 'CH_{}_model'.format(ch), basedir=pathlib.Path(self.out_dir) / 'models') # Show learning curve and example validation results try: history = model.train(X, Y, validation_data=(X_val, Y_val)) except tf.errors.ResourceExhaustedError: print( "ResourceExhaustedError: Aborting...\n Training data too big for GPU. Are other GPU jobs running? Perhaps, reduce batch-size or patch-size?" ) return #print(sorted(list(history.history.keys()))) plt.figure(figsize=(16, 5)) plot_history(history, ['loss', 'val_loss'], ['mse', 'val_mse', 'mae', 'val_mae']) plt.figure(figsize=(12, 7)) _P = model.keras_model.predict(X_val[:5]) plot_some(X_val[:5], Y_val[:5], _P, pmax=99.5, cmap="gray") plt.suptitle('5 example validation patches\n' 'top row: input (source), ' 'middle row: target (ground truth), ' 'bottom row: predicted from source') plt.show() print("-- Export model for use in Fiji...") model.export_TF() print("Done")
def dev(args): import json # Load and parse training data (X, Y), (X_val, Y_val), axes = load_training_data(args.train_data, validation_split=args.valid_split, axes=args.axes, verbose=True) c = axes_dict(axes)['C'] n_channel_in, n_channel_out = X.shape[c], Y.shape[c] # Model config print('args.resume: ', args.resume) if args.resume: # If resuming, config=None will reload the saved config config = None print('Attempting to resume') elif args.config: print('loading config from args') config_args = json.load(open(args.config)) config = Config(**config_args) else: config = Config(axes, n_channel_in, n_channel_out, probabilistic=args.prob, train_steps_per_epoch=args.steps, train_epochs=args.epochs) print(vars(config)) # Load or init model model = CARE(config, args.model_name, basedir='models') # Training, tensorboard available history = model.train(X, Y, validation_data=(X_val, Y_val)) # Plot training results print(sorted(list(history.history.keys()))) plt.figure(figsize=(16, 5)) plot_history(history, ['loss', 'val_loss'], ['mse', 'val_mse', 'mae', 'val_mae']) plt.savefig(args.model_name + '_training.png') # Export model to be used w/ csbdeep fiji plugins and KNIME flows model.export_TF()
n_channel_in, n_channel_out, unet_n_depth=4, train_epochs=100, train_steps_per_epoch=400, train_batch_size=16, train_reduce_lr={ 'patience': 5, 'factor': 0.5 }) print(config) vars(config) # In[5]: model = CARE(config=config, name=ModelName, basedir=BaseDir) # In[6]: history = model.train(X, Y, validation_data=(X_val, Y_val)) # In[7]: # In[8]: # In[9]: model.export_TF() # In[10]:
def train(X, Y, X_val, Y_val, axes, model_name, csv_file,probabilistic, validation_split, patch_size, **kwargs): """Trains CARE model with patches previously created. CARE model parameters configurated via 'Config' object: * parameters of the underlying neural network, * the learning rate, * the number of parameter updates per epoch, * the loss function, and * whether the model is probabilistic or not. Parameters ---------- X : np.array Input data X for training Y : np.array Ground truth data Y for training X_val : np.array Input data X for validation Y_val : np.array Ground truth Y for validation axes : str Semantic order of the axis in the image train_steps_per_epoch : int Number of training steps per epochs train_epochs : int Number of training epochs model_name : str Name of the model to be saved after training Returns ------- history Object with the loss values saved """ config = Config(axes, n_channel_in=1, n_channel_out=1, probabilistic=probabilistic, allow_new_parameters=True, **kwargs) model = CARE(config, model_name, basedir='models') # # Training # [TensorBoard](https://www.tensorflow.org/programmers_guide/summaries_and_tensorboard) from the current working directory with `tensorboard --logdir=.` # Then connect to [http://localhost:6006/](http://localhost:6006/) with your browser. history = model.train(X, Y, validation_data=(X_val, Y_val)) # df = pd.DataFrame() # df['probabilistic'] = [config.probabilistic] # df['batch_size'] = config.train_batch_size # df['train_epochs'] = config.train_epochs # df['lr'] = config.train_learning_rate # df['train_steps_per_epoch'] = config.train_steps_per_epoch # df['unet_last_activation'] = config.unet_last_activation # df['unet_n_depth'] = config.unet_n_depth # df['unet_n_first'] = config.unet_n_first # df['unet_residual'] = config.unet_residual # df['patch_size'] = str(patch_size) # df['validation_split'] = validation_split # last_value = len(history.history['loss']) - 1 # #dict = kwargs # #data = dict.update({'loss':history.history['loss'][last_value], }) # df['val_loss'] = history.history['val_loss'][last_value] # df['loss'] = history.history['loss'][last_value] # df['mse']=history.history['mse'][last_value] # df['mae']=history.history['mae'][last_value] # df['val_mse']=history.history['val_mse'][last_value] # df['val_mae']=history.history['val_mae'][last_value] # #df = pd.DataFrame.from_dict(data) # #print('File saved in:', os.getcwd()+'/'+csv_file) # df.to_csv('/data/'+csv_file) model.export_TF() return history
def training(self): #Loading Files and Plot examples basepath = 'data/' training_original_dir = 'training/original/' training_ground_truth_dir = 'training/ground_truth/' validation_original_dir = 'validation/original/' validation_ground_truth_dir = 'validation/ground_truth/' import glob from skimage import io from matplotlib import pyplot as plt training_original_files = sorted( glob.glob(basepath + training_original_dir + '*.tif')) training_original_file = io.imread(training_original_files[0]) training_ground_truth_files = sorted( glob.glob(basepath + training_ground_truth_dir + '*.tif')) training_ground_truth_file = io.imread(training_ground_truth_files[0]) print("Training dataset's number of files and dimensions: ", len(training_original_files), training_original_file.shape, len(training_ground_truth_files), training_ground_truth_file.shape) training_size = len(training_original_file) validation_original_files = sorted( glob.glob(basepath + validation_original_dir + '*.tif')) validation_original_file = io.imread(validation_original_files[0]) validation_ground_truth_files = sorted( glob.glob(basepath + validation_ground_truth_dir + '*.tif')) validation_ground_truth_file = io.imread( validation_ground_truth_files[0]) print("Validation dataset's number of files and dimensions: ", len(validation_original_files), validation_original_file.shape, len(validation_ground_truth_files), validation_ground_truth_file.shape) validation_size = len(validation_original_file) if training_size == validation_size: size = training_size else: print( 'Training and validation images should be of the same dimensions!' ) plt.figure(figsize=(16, 4)) plt.subplot(141) plt.imshow(training_original_file) plt.subplot(142) plt.imshow(training_ground_truth_file) plt.subplot(143) plt.imshow(validation_original_file) plt.subplot(144) plt.imshow(validation_ground_truth_file) #preparing inputs for NN from pairs of 32bit TIFF image files with intensities in range 0..1. . Run it only once for each new dataset from csbdeep.data import RawData, create_patches, no_background_patches training_data = RawData.from_folder( basepath=basepath, source_dirs=[training_original_dir], target_dir=training_ground_truth_dir, axes='YX', ) validation_data = RawData.from_folder( basepath=basepath, source_dirs=[validation_original_dir], target_dir=validation_ground_truth_dir, axes='YX', ) # pathces will be created further in "data augmentation" step, # that's why patch size here is the dimensions of images and number of pathes per image is 1 size1 = 64 X, Y, XY_axes = create_patches( raw_data=training_data, patch_size=(size1, size1), patch_filter=no_background_patches(0), n_patches_per_image=1, save_file=basepath + 'training.npz', ) X_val, Y_val, XY_axes = create_patches( raw_data=validation_data, patch_size=(size1, size1), patch_filter=no_background_patches(0), n_patches_per_image=1, save_file=basepath + 'validation.npz', ) #Loading training and validation data into memory from csbdeep.io import load_training_data (X, Y), _, axes = load_training_data(basepath + 'training.npz', verbose=False) (X_val, Y_val), _, axes = load_training_data(basepath + 'validation.npz', verbose=False) X.shape, Y.shape, X_val.shape, Y_val.shape from csbdeep.utils import axes_dict c = axes_dict(axes)['C'] n_channel_in, n_channel_out = X.shape[c], Y.shape[c] batch = len( X ) # You should define number of batches according to the available memory #batch=1 seed = 1 from keras.preprocessing.image import ImageDataGenerator data_gen_args = dict(samplewise_center=False, samplewise_std_normalization=False, zca_whitening=False, fill_mode='reflect', rotation_range=30, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, vertical_flip=True #width_shift_range=0.2, #height_shift_range=0.2, ) # training image_datagen = ImageDataGenerator(**data_gen_args) mask_datagen = ImageDataGenerator(**data_gen_args) image_datagen.fit(X, augment=True, seed=seed) mask_datagen.fit(Y, augment=True, seed=seed) image_generator = image_datagen.flow(X, batch_size=batch, seed=seed) mask_generator = mask_datagen.flow(Y, batch_size=batch, seed=seed) generator = zip(image_generator, mask_generator) # validation image_datagen_val = ImageDataGenerator(**data_gen_args) mask_datagen_val = ImageDataGenerator(**data_gen_args) image_datagen_val.fit(X_val, augment=True, seed=seed) mask_datagen_val.fit(Y_val, augment=True, seed=seed) image_generator_val = image_datagen_val.flow(X_val, batch_size=batch, seed=seed) mask_generator_val = mask_datagen_val.flow(Y_val, batch_size=batch, seed=seed) generator_val = zip(image_generator_val, mask_generator_val) # plot examples x, y = generator.__next__() x_val, y_val = generator_val.__next__() plt.figure(figsize=(16, 4)) plt.subplot(141) plt.imshow(x[0, :, :, 0]) plt.subplot(142) plt.imshow(y[0, :, :, 0]) plt.subplot(143) plt.imshow(x_val[0, :, :, 0]) plt.subplot(144) plt.imshow(y_val[0, :, :, 0]) import os blocks = 2 channels = 16 learning_rate = 0.0004 learning_rate_decay_factor = 0.95 epoch_size_multiplicator = 20 # basically, how often do we decrease learning rate epochs = 10 #comment='_side' # adds to model_name import datetime #model path model_path = f'models/CSBDeep/model.h5' if os.path.isfile(model_path): print('Your model will be overwritten in the next cell') kernel_size = 3 from csbdeep.models import Config, CARE from keras import backend as K best_mae = 1 steps_per_epoch = len(X) * epoch_size_multiplicator validation_steps = len(X_val) * epoch_size_multiplicator if 'model' in globals(): del model if os.path.isfile(model_path): os.remove(model_path) for i in range(epochs): print('Epoch:', i + 1) learning_rate = learning_rate * learning_rate_decay_factor config = Config(axes, n_channel_in, n_channel_out, unet_kern_size=kernel_size, train_learning_rate=learning_rate, unet_n_depth=blocks, unet_n_first=channels) model = CARE(config, '.', basedir='models') #os.remove('models/config.json') if i > 0: model.keras_model.load_weights(model_path) model.prepare_for_training() history = model.keras_model.fit_generator( generator, validation_data=generator_val, validation_steps=validation_steps, epochs=1, verbose=0, shuffle=True, steps_per_epoch=steps_per_epoch) if history.history['val_mae'][0] < best_mae: best_mae = history.history['val_mae'][0] if not os.path.exists('models/'): os.makedirs('models/') model.keras_model.save(model_path) print(f'Validation MAE:{best_mae:.3E}') del model K.clear_session() #model path model_path = 'models/CSBDeep/model.h5' config = Config(axes, n_channel_in, n_channel_out, unet_kern_size=kernel_size, train_learning_rate=learning_rate, unet_n_depth=blocks, unet_n_first=channels) model = CARE(config, '.', basedir='models') model.keras_model.load_weights(model_path) model.export_TF()
def train(self, channels=None, **config_args): # limit_gpu_memory(fraction=1) if channels is None: channels = self.train_channels with Timer("Training"): for ch in channels: print("-- Training channel {}...".format(ch)) (X, Y), (X_val, Y_val), axes = load_training_data( self.get_training_patch_path() / "CH_{}_training_patches.npz".format(ch), validation_split=0.1, verbose=False, ) c = axes_dict(axes)["C"] n_channel_in, n_channel_out = X.shape[c], Y.shape[c] config = Config( axes, n_channel_in, n_channel_out, train_epochs=self.train_epochs, train_steps_per_epoch=self.train_steps_per_epoch, train_batch_size=self.train_batch_size, probabilistic=self.probabilistic, **config_args, ) # Training # if ( # pathlib.Path(self.out_dir) / "models" / "CH_{}_model".format(ch) # ).exists(): # print("config there already") # config = None model = CARE( config, "CH_{}_model".format(ch), basedir=pathlib.Path(self.out_dir) / "models", ) # Show learning curve and example validation results try: history = model.train(X, Y, validation_data=(X_val, Y_val)) except tf.errors.ResourceExhaustedError: print( " >> ResourceExhaustedError: Aborting...\n Training data too big for GPU. Are other GPU jobs running? Perhaps, reduce batch-size or patch-size?" ) return except tf.errors.UnknownError: print( " >> UnknownError: Aborting...\n No enough memory available on GPU... are other GPU jobs running?" ) return # print(sorted(list(history.history.keys()))) plt.figure(figsize=(16, 5)) plot_history(history, ["loss", "val_loss"], ["mse", "val_mse", "mae", "val_mae"]) plt.figure(figsize=(12, 7)) _P = model.keras_model.predict(X_val[:5]) if self.probabilistic: _P = _P[..., 0] plot_some(X_val[:5], Y_val[:5], _P, pmax=99.5, cmap="gray") plt.suptitle("5 example validation patches\n" "top row: input (source), " "middle row: target (ground truth), " "bottom row: predicted from source") plt.show() print("-- Export model for use in Fiji...") model.export_TF() print("Done")