def train_network(optimiser='rmsprop', no_conv_layers=5, no_hidden_layers=4): filename_prefix = "%s_%s_%i_%i_%s" % (args.network_prefix, optimiser, no_conv_layers, no_hidden_layers, str(int(time.time()))) batch_size = 16 num_epochs = 2000 # we iterate 2000 times over the entire training set kernel_size = 3 # we will use 3x3 kernels throughout pool_size = 2 # we will use 2x2 pooling throughout conv_depth_1 = 32 # we will initially have 32 kernels per conv. layer... conv_depth_2 = 64 # ...switching to 64 after the first pooling layer drop_prob_1 = 0.5 # dropout after pooling with probability 0.25 drop_prob_2 = 0.5 # dropout in the FC layer with probability 0.5 hidden_size_2 = 512 earlystop_p = 5 #(X_train, y_train), (X_test, y_test) = cifar10.load_data() # fetch CIFAR-10 data input_data = np.load(args.input_file) X_train = input_data['X_train'] y_train = input_data['y_train'] X_test = input_data['X_test'] y_test = input_data['y_test'] X_train, y_train = shuffle(X_train, y_train, random_state=0) try: num_train, width, depth = X_train.shape # there are 50000 training examples in CIFAR-10 except ValueError: depth = 1 num_train, width = X_train.shape print("Single Channel Detected") num_test = X_test.shape[0] # there are 10000 test examples in CIFAR-10 num_classes = np.unique(y_train).shape[0] # there are 10 image classes X_train = X_train.astype('float32') X_test = X_test.astype('float32') #X_train /= np.max(X_train) # Normalise data to [0, 1] range #X_test /= np.max(X_test) # Normalise data to [0, 1] range X_train = norm_data(X_train) X_test = norm_data(X_test) Y_train = np_utils.to_categorical(y_train, num_classes) # One-hot encode the labels Y_test = np_utils.to_categorical(y_test, num_classes) # One-hot encode the labels if depth == 1: X_train = np.expand_dims(X_train, axis=-1) X_test = np.expand_dims(X_test, axis=-1) ### Set up CNN model ## inp = Input(shape=( width, depth)) # depth goes last in TensorFlow back-end (first in Theano) # Conv [32] -> Conv [32] -> Pool (with dropout on the pooling layer) x = Convolution1D(conv_depth_1, (2 * kernel_size), padding='same', activation='relu')(inp) x = Convolution1D(conv_depth_1, (2 * kernel_size), padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=(pool_size))(x) x = Dropout(drop_prob_1)(x) for i in range(no_conv_layers - 1): # Conv [64] -> Conv [64] -> Pool (with dropout on the pooling layer) x = Convolution1D(conv_depth_2, (kernel_size), padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=(pool_size))(x) x = Dropout(drop_prob_1)(x) # Now flatten to 1D, apply FC -> ReLU (with dropout) -> softmax x = Flatten()(x) for i in range(0, no_hidden_layers): x = Dense(hidden_size_2, activation='relu')(x) x = Dropout(drop_prob_2)(x) out = Dense(num_classes, activation='softmax')(x) model_final = Model( inputs=inp, outputs=out ) # To define a model, just specify its input and output layers ######################### earlystop = EarlyStopping(monitor='val_acc', min_delta=0.001, patience=earlystop_p, verbose=1, mode='auto') csv_logger = CSVLogger('training_%s.log' % filename_prefix) callbacks_list = [earlystop, csv_logger] model_final.summary() ######################## model_final.compile( loss='categorical_crossentropy', # using the cross-entropy loss function optimizer=optimiser, # using the RMS optimiser metrics=['accuracy']) # reporting the accuracy model_final.fit( X_train, Y_train, # Train the model using the training set... batch_size=batch_size, epochs=num_epochs, verbose=1, shuffle=True, callbacks=callbacks_list, validation_split=0.2) # ...holding out 15% of the data for validation #train_steps, train_batches = batch_iter(X_train, Y_train, batch_size) #valid_steps, valid_batches = batch_iter(X_test, Y_test, batch_size) #model_final.fit_generator( # train_batches, # train_steps, # epochs=num_epochs, # validation_data=valid_batches, # validation_steps=valid_steps, # callbacks=callbacks_list) scores = model_final.evaluate( X_test, Y_test, verbose=1) # Evaluate the trained model on the test set! print("\n%s: %.2f%%" % (model_final.metrics_names[1], scores[1] * 100)) # serialize model to JSON model_json = model_final.to_json() with open("%s.nn" % filename_prefix, "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model_final.save_weights("%s.h5" % filename_prefix) print("Saved model to disk") Y_predict = model_final.predict(X_test) conf_matx = confusion_matrix(Y_test.argmax(axis=1), Y_predict.argmax(axis=1)) print(conf_matx) print(model_final.metrics_names) print(scores) with open('%s.log' % filename_prefix, "a") as f: f.write("loss, acc\n") f.write("%f, %.4f%%" % (scores[0], scores[1] * 100)) f.write('\n') f.write(str(conf_matx))
from utils import load_data, preprocess_input import keras.backend as K import tensorflow as tf data_path = '../datasets/fer2013/fer2013.csv' model_save_path = '../trained_models/simpler_CNN.hdf5' faces, emotions = load_data(data_path) faces = preprocess_input(faces) num_classes = emotions.shape[1] image_size = faces.shape[1:] batch_size = 128 num_epochs = 1000 model = simple_CNN(image_size, num_classes) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) csv_logger = CSVLogger('training.log') early_stop = EarlyStopping('val_acc',patience=200,verbose=1) model_checkpoint = ModelCheckpoint(model_save_path, 'val_acc', verbose=1, save_best_only=True) model_callbacks = [early_stop, model_checkpoint, csv_logger] #keras bug K.get_session().run(tf.global_variables_initializer()) model.fit(faces,emotions,batch_size,num_epochs,verbose=1, callbacks=model_callbacks, validation_split=.1, shuffle=True)
from keras.callbacks import History from keras.callbacks import ModelCheckpoint history = History() # 设置log的存储位置,将网络权值以图片格式保持在tensorboard中显示,设置每一个周期计算一次网络的 tb_cb = keras.callbacks.TensorBoard(log_dir='weights/' + args.exp_name + '/%s/log' % args.model_name, write_images=1, histogram_freq=0) # 模型回调函数 early_stop = EarlyStopping('loss', min_delta=0.1, patience=patience, verbose=1) reduce_lr = ReduceLROnPlateau('loss', factor=0.01, patience=int(patience / 2), verbose=1) csv_logger = CSVLogger(log_file_path, append=False) model_names = os.path.join(train_save_path, '%s.{epoch:02d}-{acc:2f}.hdf5' % (args.model_name)) model_checkpoint = ParallelModelCheckpoint(model, filepath=model_names, monitor='val_acc', save_best_only=True, save_weights_only=True, mode='max') # if multi_gpus == True: # model_checkpoint = ParallelModelCheckpoint(model, filepath=model_names, # monitor='loss', # save_best_only=True, # save_weights_only=False)
model.add(Dropout(0.1)) model.add(Dense(1)) model.add(Activation('sigmoid')) print(model.get_config()) # try using different optimizers and different optimizer configs model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint( filepath="kddresults/lstm1layer/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='val_acc', mode='max') csv_logger = CSVLogger('training_set_iranalysis.csv', separator=',', append=False) model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=1000, validation_data=(X_test, y_test), callbacks=[checkpointer, csv_logger]) model.save("kddresults/lstm1layer/fullmodel/lstm1layer_model.hdf5") loss, accuracy = model.evaluate(X_test, y_test) print("\nLoss: %.2f, Accuracy: %.2f%%" % (loss, accuracy * 100)) y_pred = model.predict_classes(X_test) np.savetxt('kddresults/lstm1layer/lstm1predicted.txt', y_pred, fmt='%01d')
from keras.layers.core import Dropout from keras.models import Model from keras.layers import Input, Dense from keras.utils.np_utils import to_categorical os.environ["CUDA_VISIBLE_DEVICES"]="0" #config = tf.ConfigProto() #config.gpu_options.per_process_gpu_memory_fraction = 0.8 #session = tf.Session(config=config) # config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.75 # session = tf.Session(config=config) #reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.1, # patience=2, min_lr=0.0000001) csv_logger = CSVLogger('spie_NewArch_train_51.csv') batch_size = 50 nb_classes = 2 nb_epoch = 40 data_augmentation = True # input image dimensions img_rows, img_cols = 101,101 # The patch images are RGB. img_channels = 3 train_data_dir='./train' val_data_dir='./valid' train_samples=len(os.listdir(train_data_dir+"/label_0/"))+len(os.listdir(train_data_dir+"/label_1/")) val_samples=len(os.listdir(val_data_dir+"/label_0/"))+len(os.listdir(val_data_dir+"/label_1/"))
def train(): # setup seed for random number generators for reproducibility numpy.random.seed(RANDOM_SEED) if tensorflow_version() == 2: tensorflow.random.set_seed(RANDOM_SEED) else: tensorflow.set_random_seed(RANDOM_SEED) # setup paths mdl_dir = os.path.join(OUTPUT_DIR, 'models') log_dir = os.path.join(OUTPUT_DIR, 'logs') cpt_dir = os.path.join(OUTPUT_DIR, 'checkpoints') pro_dir = os.path.join(OUTPUT_DIR, 'progress') setup_flag = True for directory in [TRAIN_IMAGES_DIR, VALID_IMAGES_DIR]: if not os.path.isdir(directory): print('[INFO] Data directory not found at {}'.format(directory)) setup_flag = False if not os.path.isdir(PAIRS_IMAGES_DIR): print('[INFO] Data directory not found at {}'.format(directory)) for directory in [OUTPUT_DIR, mdl_dir, log_dir, cpt_dir, pro_dir]: if not os.path.isdir(directory): os.makedirs(directory) elif len(glob.glob(os.path.join(directory, '*.*'))) > 0: print('[INFO] Output directory {} must be empty'.format(directory)) setup_flag = False if not setup_flag: return mdl_file = os.path.join(mdl_dir, '{}.json'.format(ARCHITECTURE)) log_file = os.path.join(log_dir, '{}_training.csv'.format(ARCHITECTURE)) cpt_file_best = os.path.join(cpt_dir, '{}_weights.h5'.format(ARCHITECTURE)) cpt_file_last = os.path.join(cpt_dir, '{}_weights_{{}}.h5'.format(ARCHITECTURE)) # initialize messenger messenger = TelegramIM(auth_token=AUTH_TOKEN, chat_id=CHAT_ID) # initialize train data generator train_datagen = DataGenerator(source_chars=SOURCE_CHARS, target_chars=TARGET_CHARS, image_dir=TRAIN_IMAGES_DIR, image_ext=IMAGE_FILE_EXT, mode=IMAGE_READ_MODE, target_shape=INPUT_SHAPE_IMG[:2], rescale=SCALE_COEFF_IMG, batch_size=BATCH_SIZE, seed=RANDOM_SEED) # initialize valid data generator valid_datagen = DataGenerator(source_chars=SOURCE_CHARS, target_chars=TARGET_CHARS, image_dir=VALID_IMAGES_DIR, image_ext=IMAGE_FILE_EXT, mode=IMAGE_READ_MODE, target_shape=INPUT_SHAPE_IMG[:2], rescale=SCALE_COEFF_IMG, batch_size=BATCH_SIZE, seed=RANDOM_SEED) # build and serialize network print('[INFO] Building network... ', end='') fannet = FANnet(input_shapes=[INPUT_SHAPE_IMG, INPUT_SHAPE_HOT], optimizer=FUNCTION_OPTIM, loss=FUNCTION_LOSS, weights=None) print('done') fannet.summary() with open(mdl_file, 'w') as file: file.write(fannet.to_json()) # create callbacks csv_logs = CSVLogger(filename=log_file, append=True) cpt_best = ModelCheckpoint(filepath=cpt_file_best, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True) cpt_last = ModelCheckpoint(filepath=cpt_file_last.format('{epoch:d}'), monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=True) progress = ProgressMonitor(out_dir=pro_dir, charset=SOURCE_CHARS, img_dir=PAIRS_IMAGES_DIR, img_ext=IMAGE_FILE_EXT, mode=IMAGE_READ_MODE, rescale=SCALE_COEFF_IMG, thumbnail_size=(64, 64), messenger=messenger, notify_every=NOTIFY_EVERY, network_id=ARCHITECTURE.upper()) # train network fannet.fit_generator(generator=train_datagen.flow(), steps_per_epoch=train_datagen._steps, epochs=NUM_EPOCHS, callbacks=[csv_logs, cpt_best, cpt_last, progress], validation_data=valid_datagen.flow(), validation_steps=valid_datagen._steps) return
save_best_only=True, mode='max', save_weights_only=True) else: if not os.path.isdir('saved_models'): os.makedirs('saved_models') filepath = "saved_models/" + curr_time + "weights-improvement-{epoch:02d}-{val_acc:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max', save_weights_only=False) # Create csv file to log training csv_logger = CSVLogger(curr_time + 'history.csv', append=True) callbacks_list = [checkpoint, csv_logger] # Compute balanced class weights # Convert labels to integer format for sklearn's compute_class_weight y_ints = [y.argmax() for y in Y_train] class_weight = class_weight.compute_class_weight('balanced', np.unique(y_ints), y_ints) # Save weights in numpy array for use in converting to pb file if not os.path.isdir('class_weights'): os.makedirs('class_weights') np.save('class_weights/' + curr_time + '_cWeights.npy', class_weight) # Load our model
model.add(Dense(1024, input_dim=41, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(768, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(256, activation='relu')) model.add(Dropout(0.01)) model.add(Dense(5)) model.add(Activation('softmax')) # try using different optimizers and different optimizer configs model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint( filepath="kddresults/dnn4layer/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='loss') csv_logger = CSVLogger('kddresults/dnn4layer/training_set_dnnanalysis.csv', separator=',', append=False) model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=batch_size, nb_epoch=1000, callbacks=[checkpointer, csv_logger]) model.save("kddresults/dnn4layer/dnn4layer_model.hdf5")
# In[52]: import time # In[10]: start_time = time.time() # train # define optimizer and objective, compile cnn cnn.compile(loss="categorical_crossentropy", optimizer="adam",metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint(filepath="./CNN_checkpoints/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='loss') csv_logger = CSVLogger('./CNN_checkpoints/CNNtrainanalysis.csv',separator=',', append=False) cnn.fit(X_train, y_train, nb_epoch=1000,callbacks=[csv_logger,checkpointer]) cnn.save("./CNN_checkpoints/CNN_model.hdf5") end_time=time.time() print("Training Time--- %s seconds ---" % (end_time - start_time)) # In[13]: print("Training Time--- %s seconds ---" % (end_time - start_time)) # In[8]:
def make(self, cf, valid_gen): cb = [] # Jaccard callback if cf.dataset.class_mode == 'segmentation': print(' Jaccard metric') cb += [Jacc_new(cf.dataset.n_classes)] # Save image results if cf.save_results_enabled: print(' Save image result') cb += [ Save_results(n_classes=cf.dataset.n_classes, void_label=cf.dataset.void_class, save_path=cf.savepath, generator=valid_gen, epoch_length=int( math.ceil(cf.save_results_nsamples / float(cf.save_results_batch_size))), color_map=cf.dataset.color_map, classes=cf.dataset.classes, tag='valid') ] # Early stopping if cf.earlyStopping_enabled: print(' Early stopping') cb += [ EarlyStopping(monitor=cf.earlyStopping_monitor, mode=cf.earlyStopping_mode, patience=cf.earlyStopping_patience, verbose=cf.earlyStopping_verbose) ] # Define model saving callbacks if cf.checkpoint_enabled: print(' Model Checkpoint') cb += [ ModelCheckpoint( filepath=os.path.join(cf.savepath, "weights.hdf5"), verbose=cf.checkpoint_verbose, monitor=cf.checkpoint_monitor, mode=cf.checkpoint_mode, save_best_only=cf.checkpoint_save_best_only, save_weights_only=cf.checkpoint_save_weights_only) ] # Plot the loss after every epoch. if cf.plotHist_enabled: print(' Plot per epoch') cb += [ History_plot(cf.dataset.n_classes, cf.savepath, cf.train_metrics, cf.valid_metrics, cf.best_metric, cf.best_type, cf.plotHist_verbose) ] # Decay learning rate at specific epochs if cf.lrDecayScheduler_enabled: print(' Learning rate decay scheduler (Deprecated)') cb += [ LRDecayScheduler(cf.lrDecayScheduler_epochs, cf.lrDecayScheduler_rate) ] # Save the log cb += [ CSVLogger(os.path.join(cf.savepath, 'logFile.csv'), separator=',', append=False) ] # Learning rate scheduler if cf.LRScheduler_enabled: print(' Learning rate scheduler by batch') scheduler = Scheduler(cf.LRScheduler_type, cf.learning_rate, cf.LRScheduler_M, cf.LRScheduler_decay, cf.LRScheduler_S, cf.LRScheduler_power) if cf.LRScheduler_batch_epoch == 'batch': cb += [ LearningRateSchedulerBatch(scheduler.scheduler_function) ] elif cf.LRScheduler_batch_epoch == 'epoch': cb += [LearningRateScheduler(scheduler.scheduler_function)] else: raise ValueError('Unknown scheduler mode: ' + LRScheduler_batch_epoch) # TensorBoard callback if cf.TensorBoard_enabled: print(' Tensorboard') if cf.TensorBoard_logs_folder is None: log_dir = os.path.join(cf.usr_path, 'TensorBoardLogs') if not os.path.exists(log_dir): os.makedirs(log_dir) cb += [ TensorBoard(log_dir=log_dir, histogram_freq=cf.TensorBoard_histogram_freq, write_graph=cf.TensorBoard_write_graph, write_images=cf.TensorBoard_write_images) ] # ElapsedTime Callback print(' Elapsed Time') cb += [ElapsedTime()] # Output the list of callbacks return cb
lines = [ 'githash: {}'.format(githash), 'timestamp: {}'.format(time), 'mode: {}'.format(mode), 'resize: {}'.format(resize), 'version: {}'.format(version) ] file.writelines(lines) file.close ## checkpoint that saves the best weights according to the validation accuracy checkpoint = ModelCheckpoint(filepath=modelpath, monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=True) ## csv_logger to write losses and accuracies after each epoch in csv file csv_logger = CSVLogger(filename=csvpath, separator=',', append=True) print('[INFO] compiling model...') ## the top 2 (116) and 5 (86) xception blocks have been chosen to be trained, so the first 116 layers will be set immutable ## and the remaining layers will continued to be trained for layer in model.layers[:a]: layer.trainable = False for layer in model.layers[a:]: layer.trainable = True ## Adam or RMSProp with step learning rate decay: ## https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1 model.compile(optimizer=Adam(lr_schedule(0)), loss='categorical_crossentropy', metrics=['accuracy'])
model = Sequential() model.add(SimpleRNN(32,input_dim=78, return_sequences=True)) # try using a GRU instead, for fun model.add(Dropout(0.1)) model.add(SimpleRNN(32, return_sequences=True)) # try using a GRU instead, for fun model.add(Dropout(0.1)) model.add(SimpleRNN(32, return_sequences=True)) # try using a GRU instead, for fun model.add(Dropout(0.1)) model.add(SimpleRNN(32, return_sequences=False)) # try using a GRU instead, for fun model.add(Dropout(0.1)) model.add(Dense(1)) model.add(Activation('sigmoid')) # try using different optimizers and different optimizer configs model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint(filepath="results/simpleRNN3results/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True,monitor='val_acc',mode='max') csv_logger = CSVLogger('results/simpleRNN3results/training_set_iranalysis3.csv',separator=',', append=False) model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=20, validation_data=(X_test, y_test),callbacks=[checkpointer,csv_logger]) model.save("results/simpleRNN3results/lstm4layer_model.hdf5") loss, accuracy = model.evaluate(X_test, y_test) print("\nLoss: %.2f, Accuracy: %.2f%%" % (loss, accuracy*100)) y_pred = model.predict_classes(X_test) #np.savetxt('results/simpleRNN3results/lstm4predicted.txt', np.transpose([y_test,y_pred]), fmt='%01d') accuracy = accuracy_score(y_test, y_pred) recall = recall_score(y_test, y_pred , average="binary") precision = precision_score(y_test, y_pred , average="binary") f1 = f1_score(y_test, y_pred, average="binary") print("confusion matrix")
# Separate_small_data(validation_rate, test_rate) img_loader.separate_small_data(0.2, 0.1) model = nnetwork.create_vgg_model(3) model.compile('adam', loss='categorical_crossentropy', metrics=['accuracy', keras.metrics.categorical_accuracy]) patience = 70 early_stopping = EarlyStopping(monitor='val_acc', patience=patience, verbose=1) checkpointer = ModelCheckpoint(filepath='weights.hdf5', monitor='val_acc', save_best_only=True, verbose=1) # Print the batch number at the beginning of every batch. #epoch_plot_saver = LambdaCallback(on_epoch_end=lambda epoch,logs: save_plots_callback(logs) if epoch % 10 == 0 else False) csv_logger = CSVLogger('training.log', append=True) history = model.fit(x=img_loader.X_train, y=img_loader.Y_train, batch_size=64, epochs=100, validation_data=(img_loader.X_valid, img_loader.Y_valid), callbacks=[csv_logger, checkpointer, early_stopping]) score = model.evaluate(img_loader.X_test, img_loader.Y_test, verbose=1) print('Test loss:', score[0]) print('Test accuracy:', score[1]) save_plots(history)
loss=loss_function[args["loss_function"]], metrics=['accuracy']) #### fit model #### model_checkpoint = ModelCheckpoint( filepath='models/det-' + str(args["deforst"]) + '_' + args['model'] + '_' + str(args["learning_rate"]) + '-sliceX.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) csv_logger = CSVLogger(filename='LNDb_model_v1.1_sliceX', separator=',', append=True) early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0, patience=10, verbose=1) reduce_learning_rate = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=8, verbose=1, epsilon=0.001, cooldown=0, min_lr=0.0000001)
features='mfc' model_operation = 'new' # model_operations : 'new', 'load', 'test' shape = (624, 160) expected_shape = (624, 160) spect = np.zeros(shape) label = np.zeros(1) transform_for_birdvox = np.zeros((80,80)) transform_for_ff1010bird = np.zeros((80,80)) transform_for_chern = np.zeros((80,80)) transform_for_poland = np.zeros((80,80)) # Callbacks for logging during epochs reduceLR = ReduceLROnPlateau(factor=0.2, patience=5, min_lr=0.00001) checkPoint = ModelCheckpoint(filepath = checkpoint_model_name, monitor= 'val_acc', mode = 'max', save_best_only=True) csvLogger = CSVLogger(logfile_name, separator=',', append=False) ################################################ # # Data set selection # ################################################ # Parameters in this section can be adjusted to select different data sets to train, test, and validate on. # Keys by which we will access properties of a data set. The values assigned here are ultimately meaningless. # The 'k' prefix on these declarations signify that they will be used as keys in a dictionary. k_VAL_FILE = 'validation_file_path' k_TEST_FILE = 'test_file_path' k_TRAIN_FILE = 'train_file_path' k_VAL_SIZE = 'validate_size'
def runTrain(opts, sampleNumLimit=25): description = 'Triplet_Model' print(description) print("with parameters, Alpha: %s, Batch_size: %s, Embedded_size: %s, Epoch_num: %s, sampleNumLimit: %s"%(alpha, batch_size, emb_size, number_epoch, sampleNumLimit)) source = os.path.basename(opts.input).split('.')[0] ''' # ================================================================================ # This part is to prepare the files' index for geenrating triplet examples # and formulating each epoch inputs ''' allData, allLabel, label2IdDict, Id2Label = getClsIdDict(opts.input, sampleNumLimit, int(opts.data_dim)) all_traces = allData[:, :, np.newaxis] print("Load traces with ", all_traces.shape) print("Total size allocated on RAM : ", str(all_traces.nbytes / 1e6) + ' MB') num_classes = len(list(label2IdDict.keys())) print("number of classes: " + str(num_classes)) print('building positive pairs...') Xa_train, Xp_train = build_positive_pairs(range(0, num_classes), label2IdDict) # Gather the ids of all network traces that are used for training # This just union of two sets set(A) | set(B) all_traces_train_idx = list(set(Xa_train) | set(Xp_train)) print("X_train Anchor: ", Xa_train.shape) print("X_train Positive: ", Xp_train.shape) # Training the Triplet Model #shared_conv2 = DF(input_shape=(5000,1), emb_size=emb_size) input_shape = (allData.shape[1], 1) print('input shape is: ', input_shape) shared_conv2 = DF_model.DF(input_shape=input_shape, emb_size=emb_size) anchor = Input(input_shape, name='anchor') positive = Input(input_shape, name='positive') negative = Input(input_shape, name='negative') a = shared_conv2(anchor) p = shared_conv2(positive) n = shared_conv2(negative) # The Dot layer in Keras now supports built-in Cosine similarity using the normalize = True parameter. # From the Keras Docs: # keras.layers.Dot(axes, normalize=True) # normalize: Whether to L2-normalize samples along the dot product axis before taking the dot product. # If set to True, then the output of the dot product is the cosine proximity between the two samples. pos_sim = Dot(axes=-1, normalize=True)([a, p]) neg_sim = Dot(axes=-1, normalize=True)([a, n]) # customized loss loss = Lambda(cosine_triplet_loss, output_shape=(1,))([pos_sim, neg_sim]) model_triplet = Model(inputs=[anchor, positive, negative], outputs=loss) print(model_triplet.summary()) if opts.plotModel: from keras.utils import plot_model plot_model(model_triplet, to_file='triplet_model.png', dpi=200) sys.exit(1) # opt = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True) opt = optimizers.Adam(lr=0.001) model_triplet.compile(loss=identity_loss, optimizer=opt) print('finish compliation model') logpath = os.path.join(ResDir, 'Training_Log_{}.csv'.format(description)) csv_logger = CSVLogger(logpath, append=True, separator=';') # At first epoch we don't generate hard triplets start = time.time() gen_hard = SemiHardTripletGenerator(Xa_train, Xp_train, batch_size, all_traces, all_traces_train_idx, Id2Label, None) print('finish generate first batch of data, start training...') for epoch in range(number_epoch): model_triplet.fit_generator(generator=gen_hard.next_train(), steps_per_epoch=Xa_train.shape[0] // batch_size, # // 的意思是整除 epochs=1, verbose=1, callbacks=[csv_logger]) gen_hard = SemiHardTripletGenerator(Xa_train, Xp_train, batch_size, all_traces, all_traces_train_idx, Id2Label, shared_conv2) # For no semi-hard triplet #gen_hard = SemiHardTripletGenerator(Xa_train, Xp_train, batch_size, all_traces, all_traces_train_idx, None) end = time.time() time_last = end - start print('finishing training, train time is: {:f}'.format(time_last)) modelPath = os.path.join(modelDir, 'triplet_{}_{}.h5'.format(source, sampleNumLimit)) shared_conv2.save(modelPath) print('model save to path {}'.format(modelPath)) return modelPath, time_last
def main(): audio_filename = '369148__flying-deer-fx__music-box-the-flea-waltz.wav' sr = 8000 y, _ = librosa.load(audio_filename, sr=sr, mono=True) print y.shape min_y = np.min(y) max_y = np.max(y) # normalize y = (y - min_y) / (max_y - min_y) print y.dtype, min_y, max_y Audio(y, rate=sr) #matplotlib inline plt.figure(figsize=(30,5)) plt.plot(y[20000:20128].transpose()) plt.show() # Build a model os.environ["KERAS_BACKEND"] = "tensorflow" # so try to estimate next sample afte given (maxlen) samples maxlen = 128 # 128 / sr = 0.016 sec nb_output = 256 # resolution - 8bit encoding - output of hidden layers? latent_dim = 128 #dimensionality of the output space inputs = Input(shape=(maxlen, nb_output)) x = LSTM(latent_dim, return_sequences=True)(inputs) x = Dropout(0.2)(x) #prevents overfitting - fraction of input to drop x = LSTM(latent_dim)(x) x = Dropout(0.2)(x) output = Dense(nb_output, activation='softmax')(x) model = Model(inputs, output) #optimizer = Adam(lr=0.005) optimizer = RMSprop(lr=0.01) model.compile(loss='categorical_crossentropy', optimizer=optimizer) # try to estimate next_sample (0 -255) based on 256 previous samples step = 5 next_sample = [] samples = [] for j in tqdm(range(0, y.shape[0] - maxlen, step)): seq = y[j: j + maxlen + 1] seq_matrix = np.zeros((maxlen, nb_output), dtype=bool) for i,s in enumerate(seq): sample_ = int(s * (nb_output - 1)) # 0-255 if i < maxlen: seq_matrix[i, sample_] = True else: seq_vec = np.zeros(nb_output, dtype=bool) seq_vec[sample_] = True next_sample.append(seq_vec) samples.append(seq_matrix) samples = np.array(samples, dtype=bool) next_sample = np.array(next_sample, dtype=bool) print samples.shape, next_sample.shape csv_logger = CSVLogger('training_audio.log') escb = EarlyStopping(monitor='val_loss', patience=2, verbose=1) checkpoint = ModelCheckpoint("models/audio-{epoch:02d}-{val_loss:.2f}.hdf5", monitor='val_loss', save_best_only=True, verbose=1) #, period=2) model.fit(samples, next_sample, shuffle=True, batch_size=256, verbose=1, #initial_epoch=50, validation_split=0.3, nb_epoch=500, callbacks=[csv_logger, escb, checkpoint]) #matplotlib inline print "Training history" fig = plt.figure(figsize=(10,4)) ax1 = fig.add_subplot(1, 2, 1) plt.plot(model.history.history['loss']) ax1.set_title('loss') ax2 = fig.add_subplot(1, 2, 2) plt.plot(model.history.history['val_loss']) ax2.set_title('validation loss') seqA = [] for start in range(5000,220000,10000): seq = y[start: maxlen] seq_matrix = np.zeros((maxlen, nb_output), dtype=bool) for i,s in enumerate(seq): sample_ = int(s * (nb_output - 1)) # 0-255 seq_matrix[i, sample_] = True for i in tqdm(range(5000)): z = model.predict(seq_matrix.reshape((1,maxlen,nb_output))) s = sample(z[0], 1.0) seq = np.append(seq, s) sample_ = int(s * (nb_output - 1)) seq_vec = np.zeros(nb_output, dtype=bool) seq_vec[sample_] = True seq_matrix = np.vstack((seq_matrix, seq_vec)) # added generated note info seq_matrix = seq_matrix[1:] # scale back seq = seq * (max_y - min_y) + min_y # plot plt.figure(figsize=(30,5)) plt.plot(seq.transpose()) plt.show() display(Audio(seq, rate=sr)) print seq seqA.append(seq) #join seq data seqA2 = np.hstack(seqA) librosa.output.write_wav('data1_seq.wav', seqA2, sr)
from keras.preprocessing.image import ImageDataGenerator from keras.utils import np_utils from keras.callbacks import ModelCheckpoint from keras.callbacks import ReduceLROnPlateau from keras.callbacks import CSVLogger from keras.optimizers import Adam from nasnet import NASNetCIFAR, preprocess_input from cutout import cutout import numpy as np weights_file = 'NASNet-CIFAR-10.h5' lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.5), cooldown=0, patience=5, min_lr=0.5e-5) csv_logger = CSVLogger('NASNet-CIFAR-10.csv') model_checkpoint = ModelCheckpoint(weights_file, monitor='val_predictions_acc', save_best_only=True, save_weights_only=True, mode='max') batch_size = 128 nb_classes = 10 nb_epoch = 200 # should be 600 data_augmentation = True # input image dimensions img_rows, img_cols = 32, 32 # The CIFAR10 images are RGB. img_channels = 3
class_names=class_names, weights_path=output_weights_path, stats={}, workers=8, ) callbacks = [ checkpoint, TensorBoard(log_dir=os.path.join(output_dir, "logs"), batch_size=32), ReduceLROnPlateau(monitor="val_loss", factor=0.1, patience=5, verbose=1, mode="min", min_lr=1e-8), # auroc, CSVLogger(os.path.join(output_dir, "training_log.csv")), ] # train_counts, train_pos_counts = get_sample_counts(output_dir, "train", class_names) print("** compute class weights from training data **") # class_weights = get_class_weights(train_counts, train_pos_counts, multiply=1) print("** class_weights **") history = model_train.fit_generator( generator=train_generator, steps_per_epoch=len(train_generator) // 32, epochs=100, validation_data=validation_generator, validation_steps=len(validation_generator) // 32, callbacks=callbacks, # class_weight=class_weights, workers=8, shuffle=False,
def train(self, train, valid, weights_file, max_epoch, batch_size, nb_samples=None, log_file=None, tensorboard_dir=None): if isfile(log_file): _log.info('Saving ' + log_file + ' to ' + log_file + '.old...') copyfile(log_file, log_file + '.old') # Build assisting callbacks for printing progress and stopping if no performance improvements bestcheckpointer = ModelCheckpoint(filepath=weights_file, verbose=1, save_best_only=True) checkpointer = ModelCheckpoint( filepath=os.path.splitext(weights_file)[0] + '.{epoch:02d}-{val_loss:.3f}.hdf5', verbose=1, save_best_only=False) earlystopper = EarlyStopping(monitor='val_loss', patience=5, verbose=1) csvlogger = CSVLogger(log_file) batchlogger = BatchCSVLogger( log_file.rsplit('.', 1)[0] + "_batch." + log_file.rsplit('.', 1)[1]) # reduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, verbose=1, min_lr=1e-5) callbacks = [ bestcheckpointer, checkpointer, earlystopper, csvlogger, batchlogger ] # Keras currently will fail with OOM for tensorboard if validation data is not None if (K.backend() == 'tensorflow' and valid is None): _log.info('Run `tensorboard --logdir=' + tensorboard_dir + '` to open tensorboard at (default) 127.0.0.1:6006') tensorboard = TensorBoard(log_dir=tensorboard_dir, histogram_freq=1) callbacks.append(tensorboard) if isinstance(valid, GeneratorType): # Validation data is to be passed from generator validation_data = valid else: # Validation data is in numpy arrays validation_data = (valid.X_valid, valid.y_valid) try: _log.info('Running at most ' + str(max_epoch) + ' epochs') _log.info('Saving weights to ' + weights_file + '...') _log.info('Saving epoch logs to ' + log_file + '...') if isinstance(train, GeneratorType): # Training data is to be passed from generator self.model.fit_generator( generator=train, samples_per_epoch=nb_samples[Dataset.TRAIN.value], nb_epoch=max_epoch, validation_data=validation_data, nb_val_samples=nb_samples[Dataset.VALID.value], callbacks=callbacks) else: # Training data is in numpy arrays self.model.fit(x=train.X_train, y=train.y_train, batch_size=batch_size, nb_epoch=max_epoch, shuffle=True, validation_data=validation_data, callbacks=callbacks) except KeyboardInterrupt: # TODO let this filepath be set self._clean()
# split udacity csv data into training and validation train_data, val_data = split_train_val( csv_driving_data='data/driving_log.csv') # get network model and compile it (default Adam opt) nvidia_net = get_nvidia_model(summary=True) nvidia_net.compile(optimizer='adam', loss='mse') # json dump of model architecture with open('logs/model.json', 'w') as f: f.write(nvidia_net.to_json()) # define callbacks to save history and weights checkpointer = ModelCheckpoint( 'checkpoints/weights.{epoch:02d}-{val_loss:.3f}.hdf5') logger = CSVLogger(filename='logs/history.csv') # start the training nvidia_net.fit_generator( generator=generate_data_batch(train_data, augment_data=True, bias=CONFIG['bias']), steps_per_epoch=300 * CONFIG['batchsize'], epochs=50, validation_data=generate_data_batch(val_data, augment_data=False, bias=1.0), validation_steps=100 * CONFIG['batchsize'], callbacks=[checkpointer, logger])
#model = 'convlstm2d' saved_model = None # None or weights file class_limit = 9 # int, can be 1-101 or None load_to_memory = False # pre-load the sequences into memory data_type = 'images' #height, width, depth = 480, 640, 3 # input image size image_shape = (target_height, target_width, 1) # Helper: TensorBoard tb = TensorBoard(log_dir=os.path.join('data', 'logs', model)) # Helper: Save results. timestamp = time.time() csv_logger = CSVLogger(os.path.join('data', 'logs', model + '-' + 'training-' + \ str(timestamp) + '.log')) network = class_3dconv_clare # input_shape = (None, 96, 108, 3) #network = class_convLstm_clare # input_shape = (None, 96, 108, 3) #plot_model(network, to_file=model+'_model.png', show_shapes=True) batch_size = 1 # def train_model(data_type, image_shape, class_limit, model, batch_size,network=None, nb_epochs=100, train_list=None, test_list=None, jitter=None, output_dir=None): train_model(data_type, image_shape, 9, model, batch_size, network, nb_epochs=5000, train_list=f_train, test_list=f_test, output_dir='tmp1')
dropout_rate=config.DROPOUT_RATE) #model = convfunc.tumor_type_baseline_model(input_pixels=input_pixels,noutputs=noutputs_Y_tissue_of_origin,loss_function = LOSS_FUNCTION,dropout_rate = DROPOUT_RATE) #model = convfunc.multilabel_baseline_model(hyparameters_primary_disease=primary_disease,hyparameters_tissue_of_origin=tissue_of_origin, input_pixels=input_pixels,loss_function = LOSS_FUNCTION,dropout_rate = DROPOUT_RATE) # summarize layers print(model.summary()) # CALLBACKS to save automatically the best model we find during training: print("Defining callbacks...") print(" - Best model will be saved on file:", config.FILEMODEL) checkpoint = ModelCheckpoint(config.DATAOUTPUT + config.FILEMODEL, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') print(" - History stats of training will be save to:", config.FILEHISTORY) csv_logger = CSVLogger(config.DATAOUTPUT + config.FILEHISTORY, append=True) callbacks_list = [checkpoint, csv_logger] start = time.time() # Training the Network with custom data generator- WARNING it could take a long time model.fit( x=training_generator, #validation_data=(numpy.array(mytestdata),[Y_primary_disease_test,Y_tissue_of_origin_test]), #validation_data=(numpy.array(mytestdata),Y_primary_disease_test), validation_data=test_generator, use_multiprocessing=True, workers=config.N_CPUS, epochs=config.N_EPOCH, #validation_steps = len(test_sample_indexes), #validation_steps = 1, # // BATCH_SIZE_VALIDATION,
def train_model(data_type, image_shape, class_limit, model, batch_size, network=None, nb_epochs=100, train_list=None, test_list=None, jitter=None, output_dir=None): # Helper: Save the model. checkpointer = ModelCheckpoint( filepath=os.path.join('data', 'weights', model + '-' + data_type + \ '.{epoch:03d}-{val_acc:.3f}.hdf5'), verbose=1, save_best_only=True, monitor='val_acc') # Helper: TensorBoard tb = TensorBoard(log_dir=os.path.join('data', 'logs', model + '-tensorbard.log'), write_images=True) # Helper: Stop when we stop learning. early_stopper = EarlyStopping(monitor='val_acc', min_delta=0, patience=200, verbose=0, mode='auto') # Helper: Save results. timestamp = time.time() csv_logger = CSVLogger(os.path.join('data', 'logs', model + '-' + 'training-' + \ str(timestamp) + '.log')) fread_train = train_list.readlines() fread_test = test_list.readlines() x_train_input = [] y_train_label = [] y_classes = ["NoActivity", "Traj1", "Traj2", "Traj3", "Traj5", "Traj6", "Traj7", "Traj8", "Traj9"] for x in fread_train: a,b = x.split(";") x_train_input.append(a) y_train_label.append(y_classes.index(b.strip())) x_test_input = [] y_test_label = [] for x in fread_test: a,b = x.split(";") x_test_input.append(a) y_test_label.append(y_classes.index(b.strip())) # (samples,time, rows, cols, channels) # X: (1, 52, 15, 128, 1) means that you have only one sample that is a sequence of 52 images. # ------- Training data: ---------------------------------------------------- input_dir = "train/" sequences = [os.path.join(input_dir, f) for f in x_train_input] seq_train_x = [] f1_train = [] for index, i in enumerate(sequences): for (dirpath, dirnames, filenames) in walk(i): for x in filenames: # f1_train.extend(filenames) f1_train.append(os.path.join(dirpath, x)) seq_train_x.append(f1_train) f1_train = [] seq_length = [len(f) for f in seq_train_x] # testing to see if it works: # print(seq_train_x[0]) # print(y_train_label[0]) # print(seq_length[0]) # print(sequences[0]) # ------- Training data: ---------------------------------------------------- # ------- Testing data: ----------------------------------------------------- input_dir = "test/" sequences_test = [os.path.join(input_dir, f) for f in x_test_input] seq_test_x = [] f1_test = [] for index, i in enumerate(sequences_test): for (dirpath, dirnames, filenames) in walk(i): for x in filenames: # f1_test.extend(filenames) f1_test.append( os.path.join(dirpath, x)) seq_test_x.append(f1_test) f1_test = [] seq_length_test = [len(f) for f in seq_test_x] # for i, v0 in enumerate(seq_test_x): # for j, value in enumerate(seq_test_x[i]): # seq_test_x[i][j] = os.path.join(input_dir, value) # ------- Testing data: ----------------------------------------------------- # Generators training_generator = DataGenerator(seq_train_x, y_train_label, **params) validation_generator = DataGenerator(seq_test_x, y_test_label, **params) # Setup model and train # (samples,time, rows, cols, channels) # X: (1, 52, 15, 128, 1) means that you have only one sample that is a sequence of 52 images. input_shape = (None, target_height, target_width, channel_size) model = network(input_shape) print(net_summary(model)) # print ("(---------------------------- DEBUG ----------------------------)") # print("generator: ", generator) model.fit_generator(training_generator, epochs=nb_epochs, validation_data=validation_generator, use_multiprocessing=True, workers=5, callbacks=[tb, checkpointer, early_stopper, csv_logger])
validation_generator = validation_datagen.flow_from_directory( VALIDATION_DIR, target_size=(IMG_HEIGHT, IMG_WIDTH), batch_size=batch_size, class_mode="categorical") checkpoint = ModelCheckpoint("pretrained_model.h5", monitor='val_acc', verbose=1, save_best_only=True) early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto') csvlogger = CSVLogger('logs/transfer_learning_new.csv', separator='\t') model.fit_generator(train_generator, epochs=epochs, validation_data=validation_generator, verbose=1, callbacks=[checkpoint, early]) model.save('transfer_learning_new.h5') test_datagen = ImageDataGenerator(rescale=1. / 255) test_generator = test_datagen.flow_from_directory(TEST_DIR, target_size=(IMG_HEIGHT, IMG_WIDTH), batch_size=batch_size, class_mode='categorical') predictions = model.predict_generator(test_generator)
nb_words = min(MAX_NB_WORDS, len(word_index)) embedding_matrix = make_embedding_matrix(word_index, embeddings_index, nb_words, EMBEDDING_DIM) model = create_model_cnn() model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['acc']) time_now = str(datetime.now()) csv_filename = OUT_DIR + time_now + " csv_log-seed-{}.txt".format(SEED) callbacks = [EarlyStopping(monitor='val_loss', patience=3, verbose=1), ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, min_lr=0.001), CSVLogger(csv_filename) ] model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=N_EPOCHS, shuffle=True, batch_size=BATCH_SIZE, callbacks=callbacks, verbose=1) if EVAL_TEST: test_csv_file = pd.read_csv(TEST_DATASET, delimiter="\t") test_csv_file = test_csv_file.dropna() test_text = make_lower_case(test_csv_file['text']) test_sequences = tokenizer.texts_to_sequences(test_text) test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH) test_y = make_y(test_text[TASK])
def retrain_classifier(pre_trained_model='VGG16', pooling_mode='avg', classes=4, data_augm_enabled=False): """ConvNet as fixed feature extractor, consist of taking the convolutional base of a previously-trained network, running the new data through it, and training a new classifier on top of the output. (i.e. train only the randomly initialized top layers while freezing all convolutional layers of the original model). # Arguments pre_trained_model: one of `VGG16`, `VGG19`, `ResNet50`, `VGG16_Places365` pooling_mode: Optional pooling_mode mode for feature extraction - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling_mode will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling_mode will be applied. classes: optional number of classes to classify images into. data_augm_enabled: whether to augment the samples during training # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `pre_trained_model`, `pooling_mode` or invalid input shape. """ if not (pre_trained_model in {'VGG16', 'VGG19', 'ResNet50', 'VGG16_Places365'}): raise ValueError( 'The `pre_trained_model` argument should be either ' '`VGG16`, `VGG19`, `ResNet50`, ' 'or `VGG16_Places365`. Other models will be supported in future releases. ' ) if not (pooling_mode in {'avg', 'max', 'flatten'}): raise ValueError('The `pooling_mode` argument should be either ' '`avg` (GlobalAveragePooling2D), `max` ' '(GlobalMaxPooling2D), ' 'or `flatten` (Flatten).') # Define the name of the model and its weights if data_augm_enabled == True: filepath = bottleneck_features_dir + 'augm_bottleneck_features_' + pre_trained_model + '_' + pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5' log_filepath = logs_dir + 'augm_' + pre_trained_model + '_' + pooling_mode + '_log.csv' else: filepath = bottleneck_features_dir + 'bottleneck_features_' + pre_trained_model + '_' + pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5' log_filepath = logs_dir + pre_trained_model + '_' + pooling_mode + '_log.csv' # ModelCheckpoint checkpointer = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto', period=1, save_weights_only=True) early_stop = EarlyStopping(monitor='val_loss', patience=5, mode='auto') csv_logger = CSVLogger(log_filepath, append=True, separator=',') callbacks_list = [checkpointer, early_stop, csv_logger] input_tensor = Input(shape=(224, 224, 3)) # create the base pre-trained model for warm-up if pre_trained_model == 'VGG16': base_model = VGG16(weights='imagenet', include_top=False, input_tensor=input_tensor) elif pre_trained_model == 'VGG19': base_model = VGG19(weights='imagenet', include_top=False, input_tensor=input_tensor) elif pre_trained_model == 'ResNet50': base_model = ResNet50(weights='imagenet', include_top=False, input_tensor=input_tensor) elif pre_trained_model == 'VGG16_Places365': base_model = VGG16_Places365(weights='places', include_top=False, input_tensor=input_tensor) print('\n \n') print('[INFO] Vanilla `' + pre_trained_model + '` pre-trained convnet was successfully initialised.\n') x = base_model.output # Now we set-up transfer learning process - freeze all but the penultimate layer # and re-train the last Dense layer with `classes` number of final outputs representing probabilities for the different classes. # Build a randomly initialised classifier model to put on top of the convolutional model # both `avg`and `max`result in the same size of the Dense layer afterwards # Both Flatten and GlobalAveragePooling2D are valid options. So is GlobalMaxPooling2D. # Flatten will result in a larger Dense layer afterwards, which is more expensive # and may result in worse overfitting. But if you have lots of data, it might also perform better. # https://github.com/keras-team/keras/issues/8470 if pooling_mode == 'avg': x = GlobalAveragePooling2D(name='GAP')(x) elif pooling_mode == 'max': x = GlobalMaxPooling2D(name='GMP')(x) elif pooling_mode == 'flatten': x = Flatten(name='FLATTEN')(x) x = Dense(256, activation='relu', name='FC1')(x) # let's add a fully-connected layer # When random init is enabled, we want to include Dropout, # otherwise when loading a pre-trained HRA model we want to omit # Dropout layer so the visualisations are done properly (there is an issue if it is included) x = Dropout(0.5, name='DROPOUT')(x) # and a logistic layer with the number of classes defined by the `classes` argument predictions = Dense(classes, activation='softmax', name='PREDICTIONS')(x) # new softmax layer # this is the transfer learning model we will train model = Model(inputs=base_model.input, outputs=predictions) print( '[INFO] Randomly initialised classifier was successfully added on top of the original pre-trained conv. base. \n' ) print( '[INFO] Number of trainable weights before freezing the conv. base of the original pre-trained convnet: ' '' + str(len(model.trainable_weights))) # first: train only the top layers (which were randomly initialized) # i.e. freeze all convolutional layers of the preliminary base model for layer in base_model.layers: layer.trainable = False print( '[INFO] Number of trainable weights after freezing the conv. base of the pre-trained convnet: ' '' + str(len(model.trainable_weights))) print('\n') # compile the warm_up_model (should be done *after* setting layers to non-trainable) model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # # The attribute model.metrics_names will give you the display labels for the scalar outputs. # print warm_up_model.metrics_names if data_augm_enabled: print( '[INFO] Using augmented samples for training. This may take a while ! \n' ) t = now() history = model.fit_generator(augmented_train_generator, epochs=epochs, steps_per_epoch=steps_per_epoch, validation_data=val_generator, validation_steps=validation_steps, callbacks=callbacks_list) print( '[INFO] Training time for re-training the last dense layer using augmented samples: %s' % (now() - t)) elapsed_time = now() - t else: t = now() history = model.fit_generator(train_generator, epochs=epochs, steps_per_epoch=steps_per_epoch, validation_data=val_generator, validation_steps=validation_steps, callbacks=callbacks_list) print('[INFO] Training time for re-training the last dense layer: %s' % (now() - t)) elapsed_time = now() - t print('\n') # summarize history for accuracy plt.plot(history.history['acc']) plt.plot(history.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() # summarize history for loss plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('model loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() elapsed_time_entry = pre_trained_model + '_' + pooling_mode + ': ' + str( elapsed_time) file = open('elapsed_time.txt', 'a+') file.write(elapsed_time_entry) file.close() return model, elapsed_time
#compile the model conv_model.compile(optimizer='adam', loss='mean_squared_error', metrics=['mean_squared_error']) ### print summary of the model architecture conv_model.summary() datagen = ImageDataGenerator(rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, vertical_flip=True) # compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied) datagen.fit(sg_img_list) csv_logger = CSVLogger('conv_model_training.log') ckpt_conv_callback_no_loss = ModelCheckpoint( filepath='model_checkpoints/conv_weights_loss_logger.{epoch:04d}.hdf5', monitor='loss') # fits the model on batches with real-time data augmentation: conv_model_history_logger = conv_model.fit_generator( datagen.flow(sg_img_list, sg_vec_list, batch_size=128), steps_per_epoch=len(sg_img_list) / 128, epochs=2000, callbacks=[ckpt_conv_callback_no_loss, csv_logger])
checkpoint = ModelCheckpoint('models/vgg16_untrained_trainable.model', monitor='val_loss', verbose=1, save_best_only=True, mode='min', save_weights_only=False, period=1) earlystop = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=30, verbose=1, mode='auto') csvlogger = CSVLogger( filename="results/vgg16_untrained_trainable_training_csv.log", separator=",", append=False) reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, mode='auto') callbacks = [checkpoint, csvlogger, reduce] from keras.preprocessing.image import ImageDataGenerator train_datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True)
print(X_valid.shape, Y_valid.shape) # In[ ]: history = model.fit_generator(data_generator(X, Y, train_or_test='train', batch_size=1), samples_per_epoch=99, max_q_size=1, validation_data=(X_valid, Y_valid), #validation_data=data_generator(X, Y, train_or_test='test', batch_size=1), #nb_val_samples=100, nb_epoch=500, verbose=1, callbacks=[CSVLogger(folder(weights_folder) + '/' + 'training_log.csv'), ReduceLROnPlateau(monitor='val_loss', mode='min', factor=0.5, verbose=1, patience=40), ModelCheckpoint(folder(weights_folder) + '/' + \ #'weights.ep-{epoch:02d}-val_mean_IOU-{val_mean_IOU_gpu:.2f}_val_loss_{val_loss:.2f}.hdf5', 'last_checkpoint.hdf5', monitor='val_loss', mode='min', save_best_only=True, save_weights_only=False, verbose=0)]) # ### Comprehensive visual check # In[49]: pred_iou, pred_dice = [], [] for i, img_no in enumerate(test_idx):