def get_callbacks(args): callbacks = [] if args.tensorboard: log_dir = "logs/fit/" + datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) callbacks.append(tensorboard_callback) if args.nvprof: callbacks.append( CudaProfileCallback(args.nvprof_epoch, args.nvprof_start, args.nvprof_stop)) # Enable TFLMS if args.lms: # Specifying this starting name, from previous runs of LMS, # speeds up graph analysis time. starting_names = ['conv1_bn/cond/pred_id'] lms = LMSKerasCallback(n_tensors=args.n_tensors, lb=args.lb, starting_op_names=starting_names) callbacks.append(lms) return callbacks
def get_callbacks(args): callbacks = [] if args.nvprof: callbacks.append(CudaProfileCallback(args.nvprof_epoch, args.nvprof_start, args.nvprof_stop)) # Enable TFLMS if args.lms: # Specifying this starting name, from previous runs of LMS, # speeds up graph analysis time. starting_names = ['bn_conv1/cond/pred_id'] lms = LMSKerasCallback(n_tensors=args.n_tensors, lb=args.lb, starting_op_names=starting_names) callbacks.append(lms) return callbacks
#!/usr/bin/env python import tensorflow as tf from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.keras import backend as K config = tf.ConfigProto() config.graph_options.rewrite_options.memory_optimization = rewriter_config_pb2.RewriterConfig.SCHEDULING_HEURISTICS config.graph_options.rewrite_options.dependency_optimization = rewriter_config_pb2.RewriterConfig.OFF K.set_session(tf.Session(config=config)) from tensorflow_large_model_support import LMSKerasCallback lms_callback = LMSKerasCallback() import tifffile from functions import unet_model_3d, data_gen if __name__ == "__main__": image = tifffile.imread('traindata/training_input.tif') label = tifffile.imread('traindata/training_groundtruth.tif') res = 56 # 8*n window_size = (res, res, res) input_data = data_gen(image, window_size) label_data = data_gen(label, window_size) print('image size:', image.shape) print('data size:', input_data.shape) model = unet_model_3d((1, ) + window_size) model.summary() batch_size = 8 no_epochs = 10 model.fit(input_data, label_data,
from keras.backend.tensorflow_backend import set_session config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 8.0 set_session(tf.Session(config=config)) os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" os.environ["TF_CUDA_HOST_MEM_LIMIT_IN_MB"] = "120000" os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "TRUE" # include large model support from tensorflow_large_model_support import LMSKerasCallback # LMSKerasCallback and LMS share a set of keyword arguments. Here we just # use the default options. #lms_callback = LMSKerasCallback(starting_scope = ['conv_1'],n_tensors=-1, debug=True, debug_level=2) lms_callback = LMSKerasCallback(swap_branches=False) tf.logging.set_verbosity(tf.logging.INFO) LABELS = ['spine'] IMAGE_Z, IMAGE_H, IMAGE_W = 96, 416, 416 # when changing the 96 change as well in preproceessing3d and utils3d GRID_Z, GRID_H, GRID_W = 3, 13, 13 BOX = 5 CLASS = len(LABELS) CLASS_WEIGHTS = np.ones(CLASS, dtype='float32') OBJ_THRESHOLD = 0.3 #0.5 NMS_THRESHOLD = 0.3 #0.45 ANCHORS = [ 0.5, 0.5, 0.5, 0.8, 0.8, 0.8, 0.8, 0.5, 0.5, 0.5, 0.8, 0.5, 0.5, 0.5, 0.8 ]
def train_model(training_imagesize=224, batchsize=32, epochs=25): keras.backend.clear_session() base_model = VGG16(weights='imagenet', include_top=False, input_tensor=Input(shape=(training_imagesize, training_imagesize, 3))) x = base_model.output x = GlobalAveragePooling2D()(x) x = Dense(100, activation='relu')(x) x = Dense(50, activation='relu')(x) x = Dense(10, activation='relu')(x) preds = Dense(2, activation='softmax')(x) # final layer with softmax activation model = Model(inputs=base_model.input, outputs=preds) # Assigning which layers are to be trainined/unfrozen # for layer in model.layers[:20]: # layer.trainable = True # for layer in model.layers[20:]: # layer.trainable = True train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input, rotation_range=20, zoom_range=0.15, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15, horizontal_flip=True, vertical_flip=True) train_generator = train_datagen.flow_from_directory('./data/train/', # this is where you specify the path to the main data folder target_size=(training_imagesize, training_imagesize), color_mode='rgb', batch_size=batchsize, class_mode='categorical', shuffle=True) adam_optimizer = Adam(lr=0.0001, decay=0.9) model.compile(optimizer=adam_optimizer, loss='binary_crossentropy', metrics=['accuracy']) # test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input) # included in our dependencies test_generator = test_datagen.flow_from_directory('./data/test/', target_size=(training_imagesize, training_imagesize), color_mode='rgb', batch_size=batchsize, class_mode='categorical', shuffle=True) step_size_train = train_generator.n // train_generator.batch_size #Large model support callback to control the amount of gpu ram being used to prevent out of ram error lms_callback = LMSKerasCallback() fit_history = model.fit_generator(generator=train_generator, steps_per_epoch=step_size_train, epochs=epochs, validation_data=test_generator, validation_steps=test_generator.n // test_generator.batch_size, callbacks=[lms_callback]) # Reporting of the model for analysis purposes plt.figure(1, figsize=(15, 8)) plt.subplot(221) plt.plot(fit_history.history['acc']) plt.plot(fit_history.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'valid']) plt.subplot(222) plt.plot(fit_history.history['loss']) plt.plot(fit_history.history['val_loss']) plt.title('model loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'valid']) plt.savefig("Graphical Outputs.pdf") plt.clf() # clears entire figure trainloss = fit_history.history['loss'] testloss = fit_history.history['val_loss'] trainaccuracy = fit_history.history['acc'] testaccuracy = fit_history.history['val_acc'] np.savetxt("train_loss.csv", trainloss, delimiter=",", fmt='%s') np.savetxt("test_loss.csv", testloss, delimiter=",", fmt='%s') np.savetxt("train_accuracy.csv", trainaccuracy, delimiter=",", fmt='%s') np.savetxt("test_accuracy.csv", testaccuracy, delimiter=",", fmt='%s') # Saved model which is erased from memory so there is no chance of interference model.save('animalmodel.h5') return None