def get_model_memory_usage(batch_size, model): shapes_mem_count = 0 layers = 1 for l in model.layers: print("Layer {}".format(layers)) layers+=1 single_layer_mem = 1 for s in l.output_shape: if s is None: continue single_layer_mem *= s print("Shape {} Memory {}".format(l.output_shape, single_layer_mem)) shapes_mem_count += single_layer_mem print("Total {}".format(shapes_mem_count)) trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)]) print("Trainable weights {}".format(trainable_count)) non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]) print("Non Trainable weights {}".format(non_trainable_count)) total_memory = 4.0*batch_size*(shapes_mem_count + trainable_count + non_trainable_count) gbytes = np.round(total_memory / (1024.0 ** 3), 3) return gbytes
def model_memory_params(batch_size, model): shapes_count = int(np.sum([np.prod(np.array([s if isinstance(s, int) else 1 for s in l.output_shape])) for l in model.layers])) memory = shapes_count * 4 * batch_size memory_gb = np.round(memory / (1024 ** 3), 3) print ("Memory {} GB".format(memory_gb)) trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)]) print("Trainable weights {}".format(trainable_count)) non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]) print("Non Trainable weights {}".format(non_trainable_count)) print("Total Parameters {}".format(trainable_count + non_trainable_count))
def __init__(self, layers, weights=None, dense_activation='tanh', verbose=0): self.model = self._build_model( layers, dense_activation) if weights: self.model.set_weights( weights ) self.trainable_params = int(np.sum([K.count_params(p) for p in set(self.model.trainable_weights)])) if verbose: self.model.summary()
def train(self, x_df, y_df, epochs=100, validation_split=0.3, batch_size=10, loss='mean_squared_error', shuffle=False): self.model.compile(loss=loss, optimizer=self.optimizer) self.trainable_count = int(np.sum( [K.count_params(p) for p in set(self.model.trainable_weights)])) start = time.time() if self.verbose: print('Start training (', start, ')') history = self.model.fit(x_df, y_df, batch_size=batch_size, verbose=self.verbose, epochs=epochs, validation_split=validation_split, callbacks=[self.early_stopping, self.checkpointer], shuffle=shuffle) train_time = time.time() - start if self.verbose: print('Finish trainning. Total time: ', train_time) return {'trainable_vars': self.trainable_count, 'training_time': train_time}
def get_model_memory_usage(batch_size, model): shapes_mem_count = 0 for l in model.layers: single_layer_mem = 1 for s in l.output_shape: if type(s) is tuple: s = s[1] if s is None: continue single_layer_mem *= s shapes_mem_count += single_layer_mem trainable_count = np.sum([K.count_params(p) for p in set(model.trainable_weights)]) non_trainable_count = np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]) total_memory = 4.0*batch_size*(shapes_mem_count + trainable_count + non_trainable_count) gbytes = np.round(total_memory / (1024.0 ** 3), 3) print('number of gbytes = ' + str(gbytes)) return gbytes
def __init__(self, rnn_arch=[2,16,32,64,1], drop_out=0.3, model_file="lstm_model.hdf5", new=True, min_delta = 0.0001, patience = 50, dense_activation='tanh'): """Train a RNN using the input data rnn_arch: list containing the number of neurons per layer (the number of hidden layers is defined implicitly) drop_out: drop out used in the RNN model_file: name of the file where the model is saved new: if True, a new model is created, otherwise an existent model is used """ if new: self.model = self._build_lstm_model(rnn_arch, drop_out, dense_activation) adam = Adam(lr = 5e-5) self.model.compile(loss='mean_squared_error', optimizer=adam) else: self.model = load_model(model_file) self.checkpointer = ModelCheckpoint(filepath=model_file, verbose=0, save_best_only=True) self.early_stopping = EarlyStopping(monitor='val_loss', min_delta=min_delta, patience=patience, verbose=0, mode='auto') self.trainable_count = int(np.sum([K.count_params(p) for p in set(self.model.trainable_weights)]))
def build_model(self, layers, cell=LSTM, weights=None, dense_activation='tanh', verbose=0, **kwargs): # self.hidden_layers=len(layers) - 2 # self.layers=layers # self.input_dim=layers[0] # self.output_dim=layers[-1] self.model = Sequential() for i in range(len(layers) - 2): self.model.add(cell( # Keras API 2 input_shape=(None, layers[i]), units=layers[i+1], # Keras API 1 # input_dim=layers[i], # output_dim=layers[i+1], kernel_initializer='zeros', recurrent_initializer='zeros', bias_initializer='zeros', # Uncomment to use last batch state to init next training step. # Specify shuffle=False when calling fit() # batch_size=batch_size, stateful=True, return_sequences=True if i < (len(layers) - 3) else False)) self.model.add(Dense(layers[-1], activation=dense_activation, kernel_initializer='zeros', bias_initializer='zeros')) if weights: self.model.set_weights(weights) self.trainable_params = int(np.sum( [K.count_params(p) for p in set(self.model.trainable_weights)])) if verbose > 1: self.model.summary() return self.model
current_epoch = epoch print('current epoch ' + str(current_epoch)) def on_epoch_end(self, epoch, logs=None): open('epoch/' + job_name + '.txt', 'a').close() my_callback = PrintEpoch() callbacks = [tensorboard_callback, my_callback] #[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback] # Run training if not args.resume: trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) param_dict = {} modify = False with open('param_lock.json', 'r') as fp: param_dict = json.load(fp) if job_name not in param_dict: param_dict[job_name] = trainable_count modify = True elif param_dict[job_name] != trainable_count: param_dict[job_name] = trainable_count modify = True if modify: json_file = json.dumps(param_dict) with open('param_lock.json', 'w') as fp: fp.write(json_file) os.rename('param_lock.json', 'param.json')
def count_params(weights): return int(np.sum([K.count_params(p) for p in set(weights)]))
def fine_tune(MODEL, preprocess, height, freeze_till, lr, batch, nb_epoch, weights=None): x = Input(shape=(height, height, 3)) x = Lambda(preprocess)(x) base_model = MODEL(include_top=False, input_tensor=x, weights='imagenet', pooling='avg') for layer in base_model.layers: layer.trainable = True for layer in base_model.layers[:freeze_till]: layer.trainable = False y = Dropout(0.2)(base_model.output) y = Dense(1, activation='sigmoid', kernel_initializer='he_normal')(y) model = Model(inputs=base_model.input, outputs=y, name='Transfer_Learning') sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) # print('Trainable: %d, Non-Trainable: %d' % get_params_count(model)) trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int( np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) print('Trainable: %d, Non-Trainable: %d' % (trainable_count, non_trainable_count)) model.summary() if weights is not None: model.load_weights(weights) # Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard. log_name = '/Differentiation-EP{epoch:02d}-LOSS{val_loss:.4f}.h5' log_dir = datetime.now().strftime('transfer_model_%Y%m%d_%H%M') if not os.path.exists(log_dir): os.mkdir(log_dir) rc = RocAucMetricCallback(predict_batch_size=32, include_on_batch=True) es = EarlyStopping(monitor='val_loss', patience=20, verbose=0) mc = ModelCheckpoint(log_dir + log_name, monitor='val_loss', save_best_only=True, verbose=0) tb = TensorBoard(log_dir=log_dir) history = model.fit(x=X_train, y=y_train, batch_size=batch, epochs=nb_epoch, verbose=2, validation_data=(X_val, y_val), callbacks=[es, mc, tb]) with open('log_txt', 'w') as f: f.write(str(history.history)) # plt_score(history) plt_aucScore(history) trainscore = model.evaluate(x=train, y=trainlabel) testscore = model.evaluate(x=test, y=testlabel) # train_pre = model.predict(x=train) # test_pre = model.predict(x=test) # pd.DataFrame(test_pre).to_excel(output_path) print(trainscore, testscore)
def train_models(config): # obtain x_dim x_dim = config['x_dim'] if not os.path.exists(os.path.join(config['exp_save_dir'], "models", "vis")): os.makedirs(os.path.join(config['exp_save_dir'], "models", 'vis')) logger = set_logger( os.path.join( config['exp_save_dir'], "models", 'train_model_architecture_' + dt.now().strftime("%Y-%m-%dT%H-%M-%SZ") + '.log')) print_and_log(logger, "x_dim: {}".format(x_dim)) # LTSM model architecture input_sequences = Input(shape=(config['window_size'], x_dim), name='input_sequences') lstm_1 = LSTM(250, return_sequences=True, name="lstm_1")(input_sequences) lstm_2 = LSTM(150, name="lstm_2")(lstm_1) # branching event type et_1 = Dense(128, activation='relu', name="et_1")(lstm_2) et_2 = Dropout(0.5, name="et_2")(et_1) et_3 = Dense(64, activation='relu', name="et_3")(et_2) et_4 = Dropout(0.5, name="et_4")(et_3) event_type_output = Dense(len(config['eventtype_2_id']), activation='softmax', name="event_type_output")(et_4) # branching time delay td_1 = Dense(128, activation='relu', name="td_1")(lstm_2) td_2 = Dropout(0.5, name="td_2")(td_1) td_3 = Dense(64, activation='relu', name="td_3")(td_2) td_4 = Dropout(0.5, name="td_4")(td_3) time_delay_output = Dense(1, activation='linear', name="time_delay_output")(td_4) # branching user cluster uc_1 = Dense(128, activation='relu', name="uc_1")(lstm_2) uc_2 = Dropout(0.5, name="uc_2")(uc_1) uc_3 = Dense(64, activation='relu', name="uc_3")(uc_2) uc_4 = Dropout(0.5, name="uc_4")(uc_3) user_cluster_output = Dense(config['dim_user_cluster_one_hot'], activation='softmax', name="user_cluster_output")(uc_4) # model model = Model( inputs=input_sequences, outputs=[event_type_output, time_delay_output, user_cluster_output]) model.summary(print_fn=logger.info) print(model.summary()) model = multi_gpu_model(model, gpus=config['num_gpu']) # get partition and labels with open( os.path.join(config['exp_save_dir'], "dataset", 'partition.json'), 'r') as f: partition = json.load(f) with open(os.path.join(config['exp_save_dir'], "dataset", 'labels.json'), 'r') as f: labels = json.load(f) # train val generator training_generator = DataGenerator_ET_TD_UC_one_hot( os.path.join(config['exp_save_dir'], "dataset"), partition['train'], labels, batch_size=config['batch_size'], dim=x_dim, window_size=config['window_size'], et_classes=len(config['eventtype_2_id']), uc_classes=config['dim_user_cluster_one_hot'], shuffle=config['generator_shuffle']) validation_generator = DataGenerator_ET_TD_UC_one_hot( os.path.join(config['exp_save_dir'], "dataset"), partition['validation'], labels, batch_size=config['batch_size'], dim=x_dim, window_size=config['window_size'], et_classes=len(config['eventtype_2_id']), uc_classes=config['dim_user_cluster_one_hot'], shuffle=config['generator_shuffle']) # callback # if not os.path.exists(os.path.join( # config['exp_save_dir'], "models", "vis")): # os.makedirs(os.path.join( # config['exp_save_dir'], "models", 'vis')) # TensorBoard_callback = TensorBoard( # log_dir=os.path.join(config['exp_save_dir'], "models", 'vis'), # histogram_freq=0, # write_graph=True, # write_images=True, # write_grads=False) """ Keras TensorBoard Reference: https://keras.io/callbacks/#tensorboard launch: tensorboard --logdir=/full_path_to_your_logs """ callbacks = [ # EarlyStopping(monitor='val_loss', patience=50), ModelCheckpoint(filepath=os.path.join(config['exp_save_dir'], "models", 'model.hdf5'), monitor='val_loss', verbose=2) ] # save train confg in case testing need it with open( os.path.join(config['exp_save_dir'], "models", 'train_config.pickle'), 'wb') as f: pickle.dump(config, f) print("{} saved!".format('train_config.pickle')) # model_history model_history = dict() # start training for epoch in range(config['num_epochs']): print("\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!! " "Epoch {}/{} {}".format(epoch + 1, config['num_epochs'], "---> train all branches...")) trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int( np.sum( [K.count_params(p) for p in set(model.non_trainable_weights)])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) if (config['use_user_cluster_one_hot']) and ( not config['use_user_cluster_minmax']): if config['et_loss'] == 'focalloss': model.compile(optimizer=Adam(lr=config['lr'], decay=config['decay'], amsgrad=config['amsgrad']), loss=[ categorical_focal_loss( gamma=config['focalloss_gamma'], alpha=config['focalloss_alpha']), logcosh, binary_crossentropy ], loss_weights=config['loss_weights'], metrics={ 'event_type_output': (metrics.categorical_accuracy), 'time_delay_output': (losses.mean_squared_error), 'user_cluster_output': (metrics.categorical_accuracy) }) elif config['et_loss'] == 'bce': model.compile( optimizer='adam', loss=[binary_crossentropy, logcosh, binary_crossentropy], loss_weights=config['loss_weights'], metrics={ 'event_type_output': (metrics.categorical_accuracy), 'time_delay_output': (losses.mean_squared_error), 'user_cluster_output': (metrics.categorical_accuracy) }) else: print("event type loss undefined!") pdb.set_trace() else: model.compile( optimizer='adam', loss=[ categorical_focal_loss(gamma=config['focalloss_gamma'], alpha=config['focalloss_alpha']), logcosh, logcosh ], loss_weights=config['loss_weights'], metrics={ 'event_type_output': (metrics.categorical_accuracy), 'time_delay_output': (losses.mean_squared_error), 'user_cluster_output': (losses.mean_squared_error) }) history = model.fit_generator(generator=training_generator, epochs=1, callbacks=callbacks, validation_data=validation_generator, use_multiprocessing=True, workers=config['multiprocessing_cpu'], shuffle=True) """ Whether to shuffle the order of the batches at the beginning of each epoch. Only used with instances of Sequence (keras.utils.Sequence). Has no effect when steps_per_epoch is not None. Basically, no effect here. https://stackoverflow.com/questions/49027174/ what-does-shuffle-do-in-fit-generator-in-keras """ model.save( os.path.join(config['exp_save_dir'], "models", 'model-{}.hdf5'.format(epoch + 1))) print("model-{}.hdf5 saved!".format(epoch + 1)) if len(model_history) == 0: model_history = history.history.copy() else: for key in history.history: model_history[key] += history.history[key] with open( os.path.join(config['exp_save_dir'], "models", 'history-{}.pickle'.format(epoch + 1)), 'wb') as f: pickle.dump(model_history, f) print("history-{}.pickle saved!".format(epoch + 1)) with open(os.path.join(config['exp_save_dir'], "models", 'history.pickle'), 'wb') as f: pickle.dump(model_history, f) print("history.pickle saved!") return
activation='relu', W_regularizer=l2(L2) if L2 else None)(joint) joint = Dropout(DP)(joint) joint = BatchNormalization()(joint) prediction = Dense(CLASS_COUNT, activation='softmax')(joint) model = Model(inputs=[premise, hypothesis], outputs=prediction) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() logging.info( 'Number of trainable parameters: ' + str(int(np.sum([K.count_params(p) for p in set(model.trainable_weights)])))) class LossHistory(keras.callbacks.Callback): def on_epoch_end(self, batch, logs={}): print(str(logs)) loss = logs.get('loss') val_acc = logs.get('val_acc') acc = logs.get('acc') logging.info("Epoch: " + str(batch) + "/" + str(EPOCHS) + " ACC: " + str(acc) + " LOSS: " + str(loss) + " VAL_ACC: " + str(val_acc)) logging.info('Training')
def print_parameters(self): from keras.engine.training import collect_trainable_weights logger.info("total number of parameters: %s" % self.graph.count_params()) trainable_weights = collect_trainable_weights(self.graph) total = sum([K.count_params(p) for p in trainable_weights]) logger.info("number of trainable parameters: %s" % total)
def main(): start = time.time() ap = argparse.ArgumentParser() ap.add_argument("-e", "--epochs", required=True, type=int, help="Number of epochs", default=25) ap.add_argument("-m", "--model_name", required=True, type=str, help="Imagenet model to train", default="xception") ap.add_argument("-b", "--batch_size", required=True, type=int, help="Batch size", default=32) args = ap.parse_args() # Training dataset loading train_data = np.load("train_data.npy") train_label = np.load("train_label.npy") print("Dataset Loaded...") # Train and validation split trainX, valX, trainY, valY = train_test_split(train_data, train_label, test_size=0.1, shuffle=False) print(trainX.shape, valX.shape, trainY.shape, valY.shape) # Train nad validation image data generator trainAug = ImageDataGenerator( rescale=1.0 / 255.0, rotation_range=30, zoom_range=0.15, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15, horizontal_flip=True, fill_mode="nearest", ) valAug = ImageDataGenerator(rescale=1.0 / 255.0) model = cnn_model(args.model_name) # Number of trainable and non-trainable parameters trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int( np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) print("Total params: {:,}".format(trainable_count + non_trainable_count)) print("Trainable params: {:,}".format(trainable_count)) print("Non-trainable params: {:,}".format(non_trainable_count)) if not exists("./trained_wts"): makedirs("./trained_wts") if not exists("./training_logs"): makedirs("./training_logs") if not exists("./plots"): makedirs("./plots") # Keras backend model_checkpoint = ModelCheckpoint( "trained_wts/xception_best.hdf5", monitor="val_loss", verbose=1, save_best_only=True, save_weights_only=True, ) stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=0) csv_logger = CSVLogger( "training_logs/xception.log", separator=",", append=True, ) print("Training is going to start in 3... 2... 1... ") # Model Training H = model.fit_generator( trainAug.flow(trainX, trainY, batch_size=args.batch_size), steps_per_epoch=len(trainX) // args.batch_size, validation_data=valAug.flow(valX, valY), validation_steps=len(valX) // args.batch_size, epochs=args.epochs, callbacks=[model_checkpoint, stopping, csv_logger], ) # plot the training loss and accuracy plt.style.use("ggplot") plt.figure() N = args.epochs plt.plot(np.arange(0, N), H.history["loss"], label="train_loss") plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss") plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc") plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc") plt.title("Training Loss and Accuracy on Santa/Not Santa") plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend(loc="lower left") plt.savefig("plots/training_plot.png") end = time.time() dur = end - start if dur < 60: print("Execution Time:", dur, "seconds") elif dur > 60 and dur < 3600: dur = dur / 60 print("Execution Time:", dur, "minutes") else: dur = dur / (60 * 60) print("Execution Time:", dur, "hours")
print("Regularization:", args.reg_value) print("Trainable embeddings:", args.trainable_embeddings) print() model = mpcnn_model.he_model_siamese siamese_model, encoder = model( (max_seq_length, ), filters_ga, filters_gb, args.conv2d_type, args.conv_act, args.dense_act, embedding_dims, embedding_matrices, word_to_index, max_seq_length, args.reg_value, args.hidden_units, args.trainable_embeddings, args.use_groupa, args.use_groupb, args.use_algo1, args.use_algo2, poolings_ga, poolings_gb, args.use_cos_a1, args.use_cos_a2, args.use_euc_a1, args.use_euc_a2, args.use_abs_a1, args.use_abs_a2) trainable_count = int( np.sum( [K.count_params(p) for p in set(siamese_model.trainable_weights)])) non_trainable_count = int( np.sum([ K.count_params(p) for p in set(siamese_model.non_trainable_weights) ])) print("Trainable parameters: {:,}".format(trainable_count)) print("Non-trainable parameters: {:,}".format(non_trainable_count)) print("Total parameters: {:,}".format(trainable_count + non_trainable_count)) print("Done!") print("\n") # Printing summaries if args.print_encoder: encoder.summary() if args.print_model:
def get_dimensions(self): # calculated the number of trianable weights return int( np.sum([ K.count_params(p) for p in set(self.model.trainable_weights) ]))
def Emonet(num_classes): # use sequential model to build a VGG like network emonet = Sequential() """ Convolution and Maxpool layers: Block 1 """ # Conv Layer 1:48x48x32 emonet.add( Conv2D(filters=32, kernel_size=3, padding='same', activation='linear', input_shape=(48, 48, 1))) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 2:48x48x32 emonet.add( Conv2D(filters=32, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 1 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.3)) """ Convolution and Maxpool layers: Block 2 """ # Conv Layer 3:24x24x64 emonet.add( Conv2D(filters=64, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 4:24x24x64 emonet.add( Conv2D(filters=64, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 2 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.3)) """ Convolution and Maxpool layers: Block 3 """ # Conv Layer 5:12x12x128 emonet.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 6:12x12x128 emonet.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 3 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.25)) """ Convolution and Maxpool layers: Block 4 """ # Conv Layer 7:6x6x256 emonet.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 8:6x6x256 emonet.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 4 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.25)) # Flatten emonet.add(Flatten()) # Dense layer 1: emonet.add(Dense(256, activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) emonet.add(Dropout(0.5)) # Dense layer 2: emonet.add(Dense(256, activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) emonet.add(Dropout(0.5)) # Output layer emonet.add(Dense(num_classes, activation='softmax')) trainable_count = int( np.sum([K.count_params(p) for p in set(emonet.trainable_weights)])) non_trainable_count = int( np.sum([K.count_params(p) for p in set(emonet.non_trainable_weights)])) # network summary print('\n\n---<summary>---') print( '\n Layers: \n\tConvolution2D: {0}\n\tMaxPooling2D: {1}\n\tFully Connected Layers: {2}' .format(8, 4, 2)) print('\n Total params: {:,}'.format(trainable_count + non_trainable_count)) print('\n Trainable params: {:,}'.format(trainable_count)) print('\n Non-trainable params: {:,}'.format(non_trainable_count)) print('\n\n---</summary>---') return emonet
def Emonet_extend(num_classes): """ This model is optional and bigger than the previous one. Practically, this model is less useful than the first one therefore is not called by the application. Use only for experimental purposes """ emonet = Sequential() """ Convolution and Maxpool layers: Block 1 """ # Conv Layer 1:48x48x32 emonet.add( Conv2D(filters=64, kernel_size=3, padding='same', activation='linear', input_shape=(48, 48, 1))) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 2:48x48x32 emonet.add( Conv2D(filters=64, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 1 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.2)) """ Convolution and Maxpool layers: Block 2 """ # Conv Layer 3:24x24x64 emonet.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 4:24x24x64 emonet.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 5:24x24x64 emonet.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 2 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.2)) """ Convolution and Maxpool layers: Block 3 """ # Conv Layer 6:12x12x256 emonet.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 7:12x12x256 emonet.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 8:12x12x256 emonet.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 9:12x12x256 emonet.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 3 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.3)) """ Convolution and Maxpool layers: Block 4 """ # Conv Layer 9:6x6x256 emonet.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 10:6x6x256 emonet.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 11:6x6x256 emonet.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # Conv Layer 12:6x6x256 emonet.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) # MaxPool layer: 4 emonet.add(MaxPooling2D(pool_size=(2, 2), padding='same')) emonet.add(Dropout(0.2)) # Flatten emonet.add(Flatten()) # Dense layer 1: emonet.add(Dense(2048, activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) emonet.add(Dropout(0.5)) # Dense layer 2: emonet.add(Dense(2048, activation='linear')) emonet.add(LeakyReLU(alpha=0.3)) emonet.add(BatchNormalization(axis=-1)) emonet.add(Dropout(0.5)) # Output layer emonet.add(Dense(num_classes, activation='softmax')) trainable_count = int( np.sum([K.count_params(p) for p in set(emonet.trainable_weights)])) non_trainable_count = int( np.sum([K.count_params(p) for p in set(emonet.non_trainable_weights)])) # network summary print('\n\n---<summary>---') print( '\n Layers: \n\tConvolution2D: {0}\n\tMaxPooling2D: {1}\n\tFully Connected Layers: {2}' .format(8, 4, 2)) print('\n Total params: {:,}'.format(trainable_count + non_trainable_count)) print('\n Trainable params: {:,}'.format(trainable_count)) print('\n Non-trainable params: {:,}'.format(non_trainable_count)) print('\n\n---</summary>---') return emonet
def _document_training(history, model, start_training, end_training, save_dir, save_models, training_size, validation_size, args): """ Write down the training results. Parameters ---------- history: dic Training history. model: Trained model. start_training: datetime Training start time. end_training: datetime Training end time. save_dir: str Path to the output directory. save_models: str Path to the folder for saveing the models. training_size: int Number of training samples. validation_size: int Number of validation samples. args: dic A dictionary containing all of the input parameters. Returns -------- ./output_name/history.npy: Training history. ./output_name/X_report.txt: A summary of parameters used for the prediction and perfomance. ./output_name/X_learning_curve_f1.png: The learning curve of Fi-scores. ./output_name/X_learning_curve_loss.png: The learning curve of loss. """ np.save(save_dir + '/history', history) model.save(save_dir + '/final_model.h5') model.to_json() model.save_weights(save_dir + '/model_weights.h5') fig = plt.figure() ax = fig.add_subplot(111) ax.plot(history.history['loss']) ax.plot(history.history['detector_loss']) ax.plot(history.history['picker_P_loss']) ax.plot(history.history['picker_S_loss']) try: ax.plot(history.history['val_loss'], '--') ax.plot(history.history['val_detector_loss'], '--') ax.plot(history.history['val_picker_P_loss'], '--') ax.plot(history.history['val_picker_S_loss'], '--') ax.legend([ 'loss', 'detector_loss', 'picker_P_loss', 'picker_S_loss', 'val_loss', 'val_detector_loss', 'val_picker_P_loss', 'val_picker_S_loss' ], loc='upper right') except Exception: ax.legend(['loss', 'detector_loss', 'picker_P_loss', 'picker_S_loss'], loc='upper right') plt.ylabel('Loss') plt.xlabel('Epoch') plt.grid(b=True, which='major', color='#666666', linestyle='-') fig.savefig(os.path.join(save_dir, str('X_learning_curve_loss.png'))) fig = plt.figure() ax = fig.add_subplot(111) ax.plot(history.history['detector_f1']) ax.plot(history.history['picker_P_f1']) ax.plot(history.history['picker_S_f1']) try: ax.plot(history.history['val_detector_f1'], '--') ax.plot(history.history['val_picker_P_f1'], '--') ax.plot(history.history['val_picker_S_f1'], '--') ax.legend([ 'detector_f1', 'picker_P_f1', 'picker_S_f1', 'val_detector_f1', 'val_picker_P_f1', 'val_picker_S_f1' ], loc='lower right') except Exception: ax.legend(['detector_f1', 'picker_P_f1', 'picker_S_f1'], loc='lower right') plt.ylabel('F1') plt.xlabel('Epoch') plt.grid(b=True, which='major', color='#666666', linestyle='-') fig.savefig(os.path.join(save_dir, str('X_learning_curve_f1.png'))) delta = end_training - start_training hour = int(delta / 3600) delta -= hour * 3600 minute = int(delta / 60) delta -= minute * 60 seconds = delta trainable_count = int( np.sum([K.count_params(p) for p in model.trainable_weights])) non_trainable_count = int( np.sum([K.count_params(p) for p in model.non_trainable_weights])) with open(os.path.join(save_dir, 'X_report.txt'), 'a') as the_file: the_file.write( '================== Overal Info ==============================' + '\n') the_file.write('date of report: ' + str(datetime.datetime.now()) + '\n') the_file.write('input_hdf5: ' + str(args['input_hdf5']) + '\n') the_file.write('input_csv: ' + str(args['input_csv']) + '\n') the_file.write('output_name: ' + str(args['output_name'] + '_outputs') + '\n') the_file.write( '================== Model Parameters =========================' + '\n') the_file.write('input_dimention: ' + str(args['input_dimention']) + '\n') the_file.write('cnn_blocks: ' + str(args['cnn_blocks']) + '\n') the_file.write('lstm_blocks: ' + str(args['lstm_blocks']) + '\n') the_file.write('padding_type: ' + str(args['padding']) + '\n') the_file.write('activation_type: ' + str(args['activation']) + '\n') the_file.write('drop_rate: ' + str(args['drop_rate']) + '\n') the_file.write( str('total params: {:,}'.format(trainable_count + non_trainable_count)) + '\n') the_file.write( str('trainable params: {:,}'.format(trainable_count)) + '\n') the_file.write( str('non-trainable params: {:,}'.format(non_trainable_count)) + '\n') the_file.write( '================== Training Parameters ======================' + '\n') the_file.write('mode of training: ' + str(args['mode']) + '\n') the_file.write('loss_types: ' + str(args['loss_types']) + '\n') the_file.write('loss_weights: ' + str(args['loss_weights']) + '\n') the_file.write('batch_size: ' + str(args['batch_size']) + '\n') the_file.write('epochs: ' + str(args['epochs']) + '\n') the_file.write('train_valid_test_split: ' + str(args['train_valid_test_split']) + '\n') the_file.write('total number of training: ' + str(training_size) + '\n') the_file.write('total number of validation: ' + str(validation_size) + '\n') the_file.write('monitor: ' + str(args['monitor']) + '\n') the_file.write('patience: ' + str(args['patience']) + '\n') the_file.write('multi_gpu: ' + str(args['multi_gpu']) + '\n') the_file.write('number_of_gpus: ' + str(args['number_of_gpus']) + '\n') the_file.write('gpuid: ' + str(args['gpuid']) + '\n') the_file.write('gpu_limit: ' + str(args['gpu_limit']) + '\n') the_file.write('use_multiprocessing: ' + str(args['use_multiprocessing']) + '\n') the_file.write( '================== Training Performance =====================' + '\n') the_file.write( 'finished the training in: {} hours and {} minutes and {} seconds \n' .format(hour, minute, round(seconds, 2))) the_file.write('stoped after epoche: ' + str(len(history.history['loss'])) + '\n') the_file.write('last loss: ' + str(history.history['loss'][-1]) + '\n') the_file.write('last detector_loss: ' + str(history.history['detector_loss'][-1]) + '\n') the_file.write('last picker_P_loss: ' + str(history.history['picker_P_loss'][-1]) + '\n') the_file.write('last picker_S_loss: ' + str(history.history['picker_S_loss'][-1]) + '\n') the_file.write('last detector_f1: ' + str(history.history['detector_f1'][-1]) + '\n') the_file.write('last picker_P_f1: ' + str(history.history['picker_P_f1'][-1]) + '\n') the_file.write('last picker_S_f1: ' + str(history.history['picker_S_f1'][-1]) + '\n') the_file.write( '================== Other Parameters =========================' + '\n') the_file.write('label_type: ' + str(args['label_type']) + '\n') the_file.write('augmentation: ' + str(args['augmentation']) + '\n') the_file.write('shuffle: ' + str(args['shuffle']) + '\n') the_file.write('normalization_mode: ' + str(args['normalization_mode']) + '\n') the_file.write('add_event_r: ' + str(args['add_event_r']) + '\n') the_file.write('add_noise_r: ' + str(args['add_noise_r']) + '\n') the_file.write('shift_event_r: ' + str(args['shift_event_r']) + '\n') the_file.write('drop_channel_r: ' + str(args['drop_channel_r']) + '\n') the_file.write('scale_amplitude_r: ' + str(args['scale_amplitude_r']) + '\n') the_file.write('pre_emphasis: ' + str(args['pre_emphasis']) + '\n')
def count_params(model): trainable_count = np.sum([K.count_params(w) for w in model.trainable_weights]) non_trainable_count = np.sum([K.count_params(w) for w in model.non_trainable_weights]) return trainable_count, non_trainable_count
predictions = Dense(5, activation = "softmax", kernel_initializer = Orthogonal())(x) model = Model(inputs = base.input, outputs = predictions) ######### for layer in model.layers: layer.trainable = True ######### model.compile(optimizer = "SGD", loss = "categorical_crossentropy", metrics = ["accuracy"]) ######### trainable_params = int(np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_params = int(np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) print("\nModel Stats") print("=" * 30) print("Total Parameters: {:,}".format((trainable_params + non_trainable_params))) print("Non-Trainable Parameters: {:,}".format(non_trainable_params)) print("Trainable Parameters: {:,}\n".format(trainable_params)) ######### ### Create training data set by oversampling all classes to the size of the majority class ### ### Classes in alphabetical order; CYT = 0, FIB = 1, HOF = 2, SYN = 3, VEN = 4 ###
def get_number_of_parameters(self): trainable_count = int( np.sum([ K.count_params(p) for p in set(self.model.trainable_weights) ])) return trainable_count
def get_trainable_count(net): """https://stackoverflow.com/questions/45046525""" return sum([K.count_params(p) for p in set(net.trainable_weights)])
def main(): start = time.time() ap = argparse.ArgumentParser() ap.add_argument("-e", "--epochs", required=True, type=int, help="Number of epochs", default=25) ap.add_argument("-w", "--weights_save_name", required=True, type=str, help="Model weights name") ap.add_argument("-b", "--batch_size", required=True, type=int, help="Batch size", default=32) args = ap.parse_args() # Training dataset loading train_data = np.load("lstm_40f_data.npy") train_label = np.load("lstm_40f_labels.npy") train_label = utils.to_categorical(train_label) print("Dataset Loaded...") # Train validation split trainX, valX, trainY, valY = train_test_split(train_data, train_label, shuffle=True, test_size=0.1) model = lstm_model(train_data) trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int( np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) # Number of trainable and non-trainable parameters print("Total params: {:,}".format(trainable_count + non_trainable_count)) print("Trainable params: {:,}".format(trainable_count)) print("Non-trainable params: {:,}".format(non_trainable_count)) # Keras backend model_checkpoint = ModelCheckpoint( "trained_wts/" + args.weights_save_name + ".hdf5", monitor="val_loss", verbose=1, save_best_only=True, save_weights_only=True, ) stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=0) print("Training is going to start in 3... 2... 1... ") # Model training H = model.fit( trainX, trainY, validation_data=(valX, valY), batch_size=args.batch_size, epochs=args.epochs, shuffle=True, callbacks=[model_checkpoint, stopping], ) # plot the training loss and accuracy plt.style.use("ggplot") plt.figure() N = stopping.stopped_epoch + 1 plt.plot(np.arange(0, N), H.history["loss"], label="train_loss") plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss") plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc") plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc") plt.title("Training Loss and Accuracy") plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend(loc="lower left") plt.savefig("plots/training_plot.png") end = time.time() dur = end - start if dur < 60: print("Execution Time:", dur, "seconds") elif dur > 60 and dur < 3600: dur = dur / 60 print("Execution Time:", dur, "minutes") else: dur = dur / (60 * 60) print("Execution Time:", dur, "hours")
def count_variables(variables): return int(np.sum([K.count_params(p) for p in set(variables)]))
def Train_frcnn( train_path, # path to the text file containing the data network_arch, # the full faster rcnn network architecture object num_epochs, # num of epochs output_weight_path, # path to save the model_all.weights as hdf5 preprocessing_function=None, config_filename="config.pickle", input_weights_path=None, train_rpn=True, train_final_classifier=True, train_base_nn=True, losses_to_watch=['rpn_cls', 'rpn_reg', 'final_cls', 'final_reg'], tb_log_dir="log", num_rois=32, horizontal_flips=False, vertical_flips=False, rot_90=False, anchor_box_scales=[128, 256, 512], anchor_box_ratios=[[1, 1], [1. / math.sqrt(2), 2. / math.sqrt(2)], [2. / math.sqrt(2), 1. / math.sqrt(2)]], im_size=600, rpn_stride=16, # depends on network architecture visualize_model=None, verify_trainable=True, optimizer_rpn=Adam(lr=1e-5), optimizer_classifier=Adam(lr=1e-5), validation_interval=3, rpn_min_overlap=0.3, rpn_max_overlap=0.7, classifier_min_overlap=0.1, classifier_max_overlap=0.5, rpn_nms_threshold=0.7, # original implementation seed=5000): """ Trains a Faster RCNN for object detection in keras NOTE: This trains 2 models namely model_rpn and model_classifer with the same shared base_nn (fixed feature extractor) Keyword Arguments train_path -- str: path to the text file or pascal_voc (no Default) network_arch --object: the full faster rcnn network .py file passed as an object (no default) num_epochs -- int: number of epochs to train (no Default) output_weight_path --str: path to save the frcnn weights (no Default) preprocessing_function --function: Optional preprocessing function (must be defined like given in keras docs) (Default None) config_filename --str: Path to save the config file. Used when testing (Default "config.pickle") input_weight_path --str: Path to hdf5 file containing weights for the model (Default None) you can pass path to both classification and detection checkpoints as long as the names dont' change train_rpn --bool: whether to train the rpn layer (Default True) train_final_classifier --bool:Whether to train the final_classifier (Fast Rcnn layer) (Default True) train_base_nn --bool:Whether to train the base_nn/fixed_feature_extractor (Default True) losses_to_watch --list: A list of losses to watch (Default ['rpn_cls','rpn_reg','final_cls','final_reg']). The losses in this list are added and then weights are saved wrt to that. The list can contain any combination of the above 4 only. tb_log_dir --str: path to log dir for tensorboard logging (Default 'log') num_rois --int: The number of rois to use at once (Default = 32) horizontal_flips --bool: augment training data by horizontal flips (Default False) vertical_flips --bool: augment training data by vertical flips (Default False) rot_90 --bool: augment training data by 90 deg rotations (Default False) anchor_box_scales --list: The list of anchor box scales to use (Default [128,256,512]) anchor_box ratios --list of list: The list of anchorbox aspect ratios to use (Default [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]) im_size --int: The size to resize the image (Default 600). This is the smallest side of Pascal VOC format rpn_stride --int: The stride for rpn (Default = 16) visualize_model --str: Path to save the model as .png file verify_trainable --bool: print layer wise names and prints if it is trainable or not (Default True) optimizer_rpn --keras.optimizer: The optimizer for rpn (Default Adam(lr=1e-5)) optimizer_classifier --keras.optimizer: The optimizer for classifier (Default Adam(lr=1e-5)) validation_interval --int: The frequency (in epochs) to do validation. supply 0 if no validation rpn_min_overlap --float: (0,1) The Min IOU in rpn layer (Default 0.3) (original implementation) rpn_max_overlap --float: (0,1) The max IOU in rpn layer (Default 0.7) (original implementation) classifier_min_overlap --float: (0,1) same as above but in final classifier (Default 0.1) (original implementation) classifier_max_overlap --float: (0,1) same as above (Default 0.5) (original implementation) rpn_nms_threshold --float :(0,1) The threshold above which to supress the bbox using Non max supression in rpn (Default 0.7)(from original implementation) seed --int: To seed the random shuffling of training data (Default = 5000) Performing alternating training: - Use the train_rpn,train_final_classifier and train_base_nn boolean arguments to accomplish alternating training. - While using the above arguments change the members of losses_to_watch = ['rpn_cls','rpn_reg','final_cls','final_reg'] accordingly else it will throw error - for eg if you are training only the base_nn and the rpn set: train_rpn = True train_base_nn = True train_final_classifier = False losses_to_watch = ['rpn_cls','rpn_reg'] (do not include 'final_cls', 'final_reg') OUTPUT: prints the training log. Does not return anything Save details: 1.saves the weights of the full FRCNN model as .h5 2.saves a tensorboard file 3.saves the history of weights saved in ./saving_log.txt so that it can be known at which epoch the model is saved 4.saves the model configuration as a .pickle file 5.optionally saves the full FRCNN architecture as .png NOTE: as of now the batch size = 1 Prints loss = 0 for losses from model which is not being trained TODO: The training is a bit slow because of the data generation step. Generate_data in multiple threads and queue them for faster training """ check_list = ['rpn_cls', 'rpn_reg', 'final_cls', 'final_reg'] for n in losses_to_watch: if n not in check_list: raise ValueError( "unsupported loss the supported losses are: {}".format( check_list)) if not train_rpn: if "rpn_cls" in losses_to_watch or "rpn_reg" in losses_to_watch: raise ValueError( "Cannot watch rpn_cls and rpn_reg when train_rpn == False") if not train_final_classifier: if "final_cls" in losses_to_watch or "final_reg" in losses_to_watch: raise ValueError( "cannot watch final_cls and final_reg when train_final_classifier == False" ) nn = network_arch random.seed(seed) np.random.seed(seed) # pass the settings from the function call, and persist them in the config object C = config.Config() C.rpn_max_overlap = rpn_max_overlap C.rpn_min_overlap = rpn_min_overlap C.classifier_min_overlap = classifier_min_overlap C.classifier_max_overlap = classifier_max_overlap C.anchor_box_scales = anchor_box_scales C.anchor_box_ratios = anchor_box_ratios C.im_size = im_size C.use_horizontal_flips = bool(horizontal_flips) C.use_vertical_flips = bool(vertical_flips) C.rot_90 = bool(rot_90) C.rpn_stride = rpn_stride C.rpn_nms_threshold = rpn_nms_threshold C.weights_all_path = output_weight_path C.num_rois = int(num_rois) # check if weight path was passed via command line if input_weights_path: C.initial_weights = input_weights_path all_imgs, classes_count, class_mapping = get_data(train_path) print("The class mapping is:") print(class_mapping) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) C.class_mapping = class_mapping print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) with open(config_filename, 'wb') as config_f: pickle.dump(C, config_f) print( 'Config has been written to {}, and can be loaded when testing to ensure correct results' .format(config_filename)) np.random.shuffle(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'train'] val_imgs = [s for s in all_imgs if s['imageset'] == 'valid'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn.nn_base(img_input, trainable=train_base_nn) # define the RPN, built on the base layers num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) rpn = nn.rpn(shared_layers, num_anchors, trainable=train_rpn) # define the classifier, built on base layers classifier = nn.classifier(shared_layers, roi_input, C.num_rois, len(classes_count), trainable=train_final_classifier) # create models model_base = Model(img_input, shared_layers) # for computing the output shape model_rpn = Model(img_input, rpn[:2]) # used for training model_classifier = Model([img_input, roi_input], classifier) # used for training # this is a model that holds both the RPN and the classifier, used to load/save and freeze/unfreeze weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) # tensorboard tbCallBack = TensorBoard(log_dir=tb_log_dir, histogram_freq=1, write_graph=False, write_images=False) tbCallBack.set_model(model_all) #NOTE: both model_rpn and model_classifer contains the base_nn try: print('loading weights from {}'.format(C.initial_weights)) model_all.load_weights(C.initial_weights, by_name=True) except: print('Could not load pretrained model weights') # number of trainable parameters trainable_count = int( np.sum([K.count_params(p) for p in set(model_all.trainable_weights)])) non_trainable_count = int( np.sum( [K.count_params(p) for p in set(model_all.non_trainable_weights)])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) if verify_trainable: for layer in model_all.layers: print(layer.name, layer.trainable) model_rpn.compile(optimizer=optimizer_rpn, loss=[ Losses.rpn_loss_cls(num_anchors), Losses.rpn_loss_regr(num_anchors) ]) model_classifier.compile( optimizer=optimizer_classifier, loss=[ Losses.class_loss_cls, Losses.class_loss_regr(len(classes_count) - 1) ], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mse') # save model_all as png for visualization if visualize_model != None: plot_model(model=model_all, to_file=visualize_model, show_shapes=True, show_layer_names=True) epoch_length = len(train_imgs) validation_epoch_length = len(val_imgs) num_epochs = int(num_epochs) iter_num = 0 # train and valid data generator data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, C, model_base, K.image_dim_ordering(), preprocessing_function, mode='train') data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, C, model_base, K.image_dim_ordering(), preprocessing_function, mode='val') losses_val = np.zeros((validation_epoch_length, 5)) losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf val_best_loss = np.Inf val_best_loss_epoch = 0 print('Starting training') def write_log(callback, names, logs, batch_no): for name, value in zip(names, logs): summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = value summary_value.tag = name callback.writer.add_summary(summary, batch_no) callback.writer.flush() train_names = [ 'train_loss_rpn_cls', 'train_loss_rpn_reg', 'train_loss_class_cls', 'train_loss_class_reg', 'train_total_loss', 'train_acc' ] val_names = [ 'val_loss_rpn_cls', 'val_loss_rpn_reg', 'val_loss_class_cls', 'val_loss_class_reg', 'val_total_loss', 'val_acc' ] for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) while True: try: if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose: mean_overlapping_bboxes = float( sum(rpn_accuracy_rpn_monitor)) / len( rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( 'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations' .format(mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.' ) X, Y, img_data = next(data_gen_train) if train_rpn: loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=C.rpn_nms_threshold, flag="train") # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou( R, img_data, C, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, C.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) if train_final_classifier: loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) # losses if train_rpn: losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] else: losses[iter_num, 0] = 0 losses[iter_num, 1] = 0 if train_final_classifier: losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] # accuracy else: losses[iter_num, 2] = 0 losses[iter_num, 3] = 0 losses[iter_num, 4] = 0 iter_num += 1 progbar.update( iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])), ('detector_cls', np.mean(losses[:iter_num, 2])), ('detector_regr', np.mean(losses[:iter_num, 3]))]) if iter_num == epoch_length: if train_rpn: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) else: loss_rpn_cls = 0 loss_rpn_regr = 0 if train_final_classifier: loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) else: loss_class_cls = 0 loss_class_regr = 0 class_acc = 0 mean_overlapping_bboxes = float(sum( rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if C.verbose: print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print( 'Classifier accuracy for bounding boxes from RPN: {}' .format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format( loss_class_cls)) print('Loss Detector regression: {}'.format( loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) loss_dict_train = { "rpn_cls": loss_rpn_cls, "rpn_reg": loss_rpn_regr, "final_cls": loss_class_cls, "final_reg": loss_class_regr } curr_loss = 0 for l in losses_to_watch: curr_loss += loss_dict_train[l] iter_num = 0 start_time = time.time() write_log(tbCallBack, train_names, [ loss_rpn_cls, loss_rpn_regr, loss_class_cls, loss_class_regr, curr_loss, class_acc ], epoch_num) if curr_loss < best_loss: if C.verbose: print( 'Total loss decreased from {} to {} in training, saving weights' .format(best_loss, curr_loss)) save_log_data = '\nTotal loss decreased from {} to {} in epoch {}/{} in training, saving weights'.format( best_loss, curr_loss, epoch_num + 1, num_epochs) with open("./saving_log.txt", "a") as f: f.write(save_log_data) best_loss = curr_loss model_all.save_weights(C.weights_all_path) break except Exception as e: print('Exception: {}'.format(e)) continue if validation_interval > 0: # validation if (epoch_num + 1) % validation_interval == 0: progbar = generic_utils.Progbar(validation_epoch_length) print("Validation... \n") while True: try: X, Y, img_data = next(data_gen_val) if train_rpn: val_loss_rpn = model_rpn.test_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi( P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=C.rpn_nms_threshold, flag="train") # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = roi_helpers.calc_iou( R, img_data, C, class_mapping) neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, C.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) if train_final_classifier: val_loss_class = model_classifier.test_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) if train_rpn: losses_val[iter_num, 0] = val_loss_rpn[1] losses_val[iter_num, 1] = val_loss_rpn[2] else: losses_val[iter_num, 0] = 0 losses_val[iter_num, 1] = 0 if train_final_classifier: losses_val[iter_num, 2] = val_loss_class[1] losses_val[iter_num, 3] = val_loss_class[2] losses_val[iter_num, 4] = val_loss_class[3] else: losses_val[iter_num, 2] = 0 losses_val[iter_num, 3] = 0 losses_val[iter_num, 4] = 0 iter_num += 1 progbar.update( iter_num, [('rpn_cls', np.mean(losses_val[:iter_num, 0])), ('rpn_regr', np.mean(losses_val[:iter_num, 1])), ('detector_cls', np.mean(losses_val[:iter_num, 2])), ('detector_regr', np.mean(losses_val[:iter_num, 3]))]) if iter_num == validation_epoch_length: if train_rpn: val_loss_rpn_cls = np.mean(losses_val[:, 0]) val_loss_rpn_regr = np.mean(losses_val[:, 1]) else: val_loss_rpn_cls = 0 val_loss_rpn_regr = 0 if train_final_classifier: val_loss_class_cls = np.mean(losses_val[:, 2]) val_loss_class_regr = np.mean(losses_val[:, 3]) val_class_acc = np.mean(losses_val[:, 4]) else: val_loss_class_cls = 0 val_loss_class_regr = 0 val_class_acc = 0 mean_overlapping_bboxes = float( sum(rpn_accuracy_for_epoch)) / len( rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] loss_dict_valid = { "rpn_cls": val_loss_rpn_cls, "rpn_reg": val_loss_rpn_regr, "final_cls": val_loss_class_cls, "final_reg": val_loss_class_regr } val_curr_loss = 0 for l in losses_to_watch: val_curr_loss += loss_dict_valid[l] write_log(tbCallBack, val_names, [ val_loss_rpn_cls, val_loss_rpn_regr, val_loss_class_cls, val_loss_class_regr, val_curr_loss, val_class_acc ], epoch_num) if C.verbose: print('[INFO VALIDATION]') print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print( 'Classifier accuracy for bounding boxes from RPN: {}' .format(val_class_acc)) print('Loss RPN classifier: {}'.format( val_loss_rpn_cls)) print('Loss RPN regression: {}'.format( val_loss_rpn_regr)) print('Loss Detector classifier: {}'.format( val_loss_class_cls)) print('Loss Detector regression: {}'.format( val_loss_class_regr)) print( "current loss: %.2f, best loss: %.2f at epoch: %d" % (val_curr_loss, val_best_loss, val_best_loss_epoch)) print('Elapsed time: {}'.format(time.time() - start_time)) if val_curr_loss < val_best_loss: if C.verbose: print( 'Total loss decreased from {} to {}, saving weights' .format(val_best_loss, val_curr_loss)) save_log_data = '\nTotal loss decreased from {} to {} in epoch {}/{} in validation, saving weights'.format( val_best_loss, val_curr_loss, epoch_num + 1, num_epochs) with open("./saving_log.txt", "a") as f: f.write(save_log_data) val_best_loss = val_curr_loss val_best_loss_epoch = epoch_num model_all.save_weights(C.weights_all_path) start_time = time.time() iter_num = 0 break except: pass print('Training complete, exiting.')
def plot_parameter_statistic(model, layer_types=['Dense', 'Conv2D'], trainable=True, non_trainable=True, outputs=False): layer_types = [l.__name__ if type(l) == type else l for l in layer_types] def get_layers_recursion(model): layers = [] for l in model.layers: if l.__class__.__name__ is 'Model': child_layers = get_layers_recursion(l) else: child_layers = [l] for cl in child_layers: if cl not in layers: layers.append(cl) return layers layers = get_layers_recursion(model) layers = [l for l in layers if l.__class__.__name__ in layer_types] names = [l.name for l in layers] y = range(len(names)) plt.figure(figsize=[12, max(len(y) // 4, 1)]) offset = np.zeros(len(layers), dtype=int) legend = [] if trainable: counts_trainable = [ np.sum([K.count_params(p) for p in set(l.trainable_weights)]) for l in layers ] plt.barh(y, counts_trainable, align='center', color='#1f77b4') offset += np.array(counts_trainable, dtype=int) legend.append('trainable') if non_trainable: counts_non_trainable = [ np.sum([K.count_params(p) for p in set(l.non_trainable_weights)]) for l in layers ] plt.barh(y, counts_non_trainable, align='center', color='#ff7f0e', left=offset) offset += np.array(counts_non_trainable, dtype=int) legend.append('non-trainable') if outputs: counts_outputs = [ np.sum([ np.sum([np.prod(s[1:]) for s in n.output_shapes]) for n in l._inbound_nodes ]) for l in layers ] plt.barh(y, counts_outputs, align='center', color='#2ca02c', left=offset) offset += np.array(counts_outputs, dtype=int) legend.append('outputs') plt.yticks(y, names) plt.ylim(y[0] - 1, y[-1] + 1) ax = plt.gca() ax.invert_yaxis() ax.xaxis.tick_top() plt.legend(legend) plt.show()
def count_parameters(model): trainable_count = int(np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int(np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) print('trainable %16i' %(trainable_count)) print('non-trainable %16i' %(non_trainable_count))
def customLoss(yTrue,yPred): if(yTrue.shape[1]!=None): return (K.mean(K.square((K.log(yTrue) - K.log(yPred))),axis=-1)-(0.5/pow(K.count_params(yTrue),2))*K.sum((K.log(yTrue) - K.log(yPred)),axis=-1)) else: return K.sum(K.abs(yPred - yTrue), axis=-1)
# top_model.load_weights('./Xception_decay.hdf5') for layer in top_model.layers: layer.trainable = True LearningRate = 0.0001 n_epochs = 30 sgd = optimizers.SGD(lr=config.MIN_LR, momentum=0.9) adam = optimizers.Adam(lr=LearningRate, decay=LearningRate / n_epochs, amsgrad=True) top_model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=["accuracy"]) trainable_params = int( np.sum([K.count_params(p) for p in set(top_model.trainable_weights)])) non_trainable_params = int( np.sum([K.count_params(p) for p in set(top_model.non_trainable_weights)])) print("\nModel Status") print("=" * 40) print("Total Parameters: {:,}".format( (trainable_params + non_trainable_params))) print("Non-Trainable Parameters: {:,}".format(non_trainable_params)) print("Trainable Parameters: {:,}\n".format(trainable_params)) train_path = '/cptjack/totem/barrylee/cut_small_cell/hepat-tri-classification/new-train/96-train' train_folders = [ train_path + "/ballooning/", train_path + "/normal/", train_path + "/steatosis/" ] #,"/cptjack/totem/barrylee/cut_small_cell/train/other_cells/"
def main(): start = time.time() epochs=30 version=2 weights_save_name = "bidir_LSTM_NT_{}".format(str(version)) batch_size=32 # Training dataset loading train_data1 = np.load("./dataset_npy/lstm_NT_train_data_500.npy", allow_pickle=True) train_label1 = np.load("./dataset_npy/lstm_NT_train_label_500.npy", allow_pickle=True) train_data2 = np.load("./dataset_npy/lstm_NT_train_data_1000.npy", allow_pickle=True) train_label2 = np.load("./dataset_npy/lstm_NT_train_label_1000.npy", allow_pickle=True) train_data = np.concatenate((train_data1, train_data2), axis=0) train_label = np.concatenate((train_label1, train_label2), axis=0) del train_data1, train_data2, train_label1, train_label2 train_data3 = np.load("./dataset_npy/lstm_NT_train_data_1500.npy", allow_pickle=True) train_label3 = np.load("./dataset_npy/lstm_NT_train_label_1500.npy", allow_pickle=True) train_data = np.concatenate((train_data, train_data3), axis=0) train_label = np.concatenate((train_label, train_label3), axis=0) del train_data3, train_label3 train_data4 = np.load("./dataset_npy/lstm_NT_train_data_1719.npy", allow_pickle=True) train_label4 = np.load("./dataset_npy/lstm_NT_train_labels_1719.npy", allow_pickle=True) train_data = np.concatenate((train_data, train_data4), axis=0) train_label = np.concatenate((train_label, train_label4), axis=0) del train_data4, train_label4 print(train_data.shape, train_label.shape) train_label = utils.to_categorical(train_label) test_data = np.load("./dataset_npy/lstm_NT_test_data.npy", allow_pickle=True) test_label = np.load("./dataset_npy/lstm_NT_test_labels.npy", allow_pickle=True) test_label = utils.to_categorical(test_label) print(test_data.shape, test_label.shape) print("Dataset Loaded...") # Train validation split trainX, valX, trainY, valY = train_test_split( train_data, train_label, shuffle=True, test_size=0.2 ) model = rnn_models(train_data) trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)]) ) non_trainable_count = int( np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]) ) # Number of trainable and non-trainable parameters print("Total params: {:,}".format(trainable_count + non_trainable_count)) print("Trainable params: {:,}".format(trainable_count)) print("Non-trainable params: {:,}".format(non_trainable_count)) # Keras backend model_checkpoint = ModelCheckpoint( "models/" + weights_save_name + ".hdf5", monitor="val_loss", verbose=1, save_best_only=True, save_weights_only=True, ) stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=0) print("Training is going to start in 3... 2... 1... ") # Model training H = model.fit( trainX, trainY, validation_data=(valX, valY), batch_size=batch_size, epochs=epochs, shuffle=True, callbacks=[model_checkpoint, stopping], ) # plot the training loss and accuracy results = model.evaluate(test_data, test_label) print("Test acc, loss : ", results) end = time.time() dur = end - start if dur < 60: print("Execution Time:", dur, "seconds") elif dur > 60 and dur < 3600: dur = dur / 60 print("Execution Time:", dur, "minutes") else: dur = dur / (60 * 60) print("Execution Time:", dur, "hours")
layer.trainable = True LearningRate = 0.01 decay = 0.0001 n_epochs = 20 sgd = optimizers.SGD(lr=LearningRate, decay=LearningRate / n_epochs, momentum=0.9, nesterov=True) parallel_model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=["accuracy"]) trainable_params = int( np.sum([K.count_params(p) for p in set(parallel_model.trainable_weights)])) non_trainable_params = int( np.sum([ K.count_params(p) for p in set(parallel_model.non_trainable_weights) ])) print("\nModel Stats") print("=" * 30) print("Total Parameters: {:,}".format( (trainable_params + non_trainable_params))) print("Non-Trainable Parameters: {:,}".format(non_trainable_params)) print("Trainable Parameters: {:,}\n".format(trainable_params)) train_folders = [ "./raw/train/cancer cell/", "./raw/train/lymphocyte/",
def test(): args = cmd.args() if args.load_generated_images: print("Loading images...") input_images, target_images, generated_images, names = load_generated_images( args.generated_images_dir) else: print("Generate images...") from keras import backend as K if args.use_dropout_test: K.set_learning_phase(1) dataset = PoseHMDataset(test_phase=True, **vars(args)) generator = make_generator(args.image_size, args.use_input_pose, args.warp_skip, args.disc_type, args.warp_agg) trainable_count = int( np.sum( [K.count_params(p) for p in set(generator.trainable_weights)])) non_trainable_count = int( np.sum([ K.count_params(p) for p in set(generator.non_trainable_weights) ])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) assert (args.generator_checkpoint is not None) generator.load_weights(args.generator_checkpoint) input_images, target_images, generated_images, names = generate_images( dataset, generator, args.use_input_pose) print("Save images to %s..." % (args.generated_images_dir, )) save_images(input_images, target_images, generated_images, names, args.generated_images_dir) print("Compute inception score...") inception_score = get_inception_score(generated_images) print("Inception score %s" % inception_score[0]) print("Compute structured similarity score (SSIM)...") structured_score = ssim_score(generated_images, target_images) print("SSIM score %s" % structured_score) print("Compute l1 score...") norm_score = l1_score(generated_images, target_images) print("L1 score %s" % norm_score) print("Compute masked inception score...") generated_images_masked = create_masked_image(names, generated_images, args.annotations_file_test) reference_images_masked = create_masked_image(names, target_images, args.annotations_file_test) inception_score_masked = get_inception_score(generated_images_masked) print("Inception score masked %s" % inception_score_masked[0]) print("Compute masked SSIM...") structured_score_masked = ssim_score(generated_images_masked, reference_images_masked) print("SSIM score masked %s" % structured_score_masked) print( "Inception score = %s, masked = %s; SSIM score = %s, masked = %s; l1 score = %s" % (inception_score, inception_score_masked, structured_score, structured_score_masked, norm_score))
n_part_param_layers = [ len(model.layers[L].trainable_weights) for L in range(n_layers) ] # how many parts of parameters that each layer has index_layers_n0 = np.nonzero(n_part_param_layers)[ 0] # indices for layers with trainable parameters n_layers_n0 = len(index_layers_n0) trainable_count = np.zeros( n_layers_n0, dtype=int) # the number of trainable parameters that each layer has for i in range(n_layers_n0): layer_id = index_layers_n0[i] trainable_count[i] = np.sum([ K.count_params(model.layers[layer_id].trainable_weights[p]) for p in range(n_part_param_layers[layer_id]) ]) trainable_count_id_start_end = np.insert(np.cumsum(trainable_count), 0, 0) t = 0 FI_train_all = np.zeros(n_train) # FI for all parameters grad_K = tf.concat( axis=1, values=[ tf.concat( axis=0, values=[ K.flatten(b)[..., None]
def get_params_count(model): trainable = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable = int( np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) return trainable, non_trainable
def evaluate(self, phenotype, load_prev_weights, weights_save_path, parent_weights_path,\ train_time, num_epochs, datagen=None, datagen_test = None, input_size=(32, 32, 3)): """ Evaluates the keras model using the keras optimiser Parameters ---------- phenotype : str individual phenotype load_prev_weights : bool resume training from a previous train or not weights_save_path : str path where to save the model weights after training parent_weights_path : str path to the weights of the previous training train_time : float maximum training time num_epochs : int maximum number of epochs datagen : keras.preprocessing.image.ImageDataGenerator Data augmentation method image data generator input_size : tuple dataset input shape Returns ------- score_history : dict training data: loss and accuracy """ model_phenotype, learning_phenotype = phenotype.split('learning:') learning_phenotype = 'learning:' + learning_phenotype.rstrip().lstrip() model_phenotype = model_phenotype.rstrip().lstrip().replace(' ', ' ') keras_layers = self.get_layers(model_phenotype) keras_learning = self.get_learning(learning_phenotype) batch_size = int(keras_learning['batch_size']) if load_prev_weights: model = keras.models.load_model( parent_weights_path.replace('.hdf5', '.h5')) else: model = self.assemble_network(keras_layers, input_size) opt = self.assemble_optimiser(keras_learning) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) #early stopping early_stop = keras.callbacks.EarlyStopping( monitor='val_loss', patience=int(keras_learning['early_stop'])) #time based stopping time_stop = TimedStopping(seconds=train_time, verbose=DEBUG) #save individual with the lowest validation loss monitor = ModelCheckpoint(weights_save_path, monitor='val_loss', verbose=0, save_best_only=True) trainable_count = int( np.sum([ backend.count_params(p) for p in set(model.trainable_weights) ])) if datagen is not None: score = model.fit_generator( datagen.flow(self.dataset['evo_x_train'], self.dataset['evo_y_train'], batch_size=batch_size), steps_per_epoch=(self.dataset['evo_x_train'].shape[0] // batch_size), epochs=int(keras_learning['epochs']), validation_data=(datagen_test.flow(self.dataset['evo_x_val'], self.dataset['evo_y_val'], batch_size=batch_size)), validation_steps=(self.dataset['evo_x_val'].shape[0] // batch_size), callbacks=[early_stop, time_stop, monitor], initial_epoch=num_epochs, verbose=DEBUG) else: score = model.fit( x=self.dataset['evo_x_train'], y=self.dataset['evo_y_train'], batch_size=batch_size, epochs=int(keras_learning['epochs']), steps_per_epoch=(self.dataset['evo_x_train'].shape[0] // batch_size), validation_data=(self.dataset['evo_x_val'], self.dataset['evo_y_val']), callbacks=[early_stop, time_stop, monitor], initial_epoch=num_epochs, verbose=DEBUG) #load weights with the lowest val loss if os.path.isfile(weights_save_path): model.load_weights(weights_save_path) #save final moodel to file model.save(weights_save_path.replace('.hdf5', '.h5')) #measure test performance if datagen_test is None: y_pred_test = model.predict(self.dataset['evo_x_test'], batch_size=batch_size, verbose=0) else: y_pred_test = model.predict_generator( datagen_test.flow(self.dataset['evo_x_test'], batch_size=100, shuffle=False), steps=self.dataset['evo_x_test'].shape[0] // 100, verbose=DEBUG) accuracy_test = self.fitness_metric(self.dataset['evo_y_test'], y_pred_test) if DEBUG: print(phenotype, accuracy_test) score.history['trainable_parameters'] = trainable_count score.history['accuracy_test'] = accuracy_test return score.history
def train(): train_dataset = dataset.H5SegmentationDataset( root=dataDir_, fn_txt=trainFileText_, n_class=numClass_, label_prob_to_cls=labelProbToCls_, label_one_hot=oneHot_, transforms=transformer.Compose([ transformer.MinMaxScale(mins=channelMin_, maxs=channelMax_), transformer.RandomGammaTransform(gamma=0.5, p=0.5), transformer.RandomBlur(kernel_size=(3, 3), p=0.1), transformer.RandomHorizontalFlip(p=0.5), transformer.RandomVerticalFlip(p=0.5), transformer.RandomRotate(angle_threshold=20, p=0.5), transformer.RandomScale(scale_range=(0.8, 1.2), p=0.5), # transformer.Normalize(mean=channelMean_, std=channelStd_) ])) val_dataset = dataset.H5SegmentationDataset( root=dataDir_, fn_txt=valFileText_, n_class=numClass_, label_prob_to_cls=labelProbToCls_, label_one_hot=oneHot_, transforms=transformer.Compose([ # transformer.Normalize(mean=channelMean_, std=channelStd_) transformer.MinMaxScale(mins=channelMin_, maxs=channelMax_) ])) train_generator = datagenerator.data_generator(train_dataset, batch_size=batchSize_, shuffle=True) val_generator = datagenerator.data_generator(val_dataset, batch_size=batchSize_, shuffle=False) stepsPerValEpoch_ = len(val_dataset) // batchSize_ stepsPerEpoch_ = stepsPerValEpoch_ * 5 # for debug if debug_: for _ in range(1): x, y = next(train_generator) print(x.shape, y.shape) if oneHot_: plot_image_label_per_channel(x[0], np.argmax(y[0], -1)) else: from semantic_segmentation.utils import plot_rgb_image plot_image_label_per_channel(x[0], y[0, :, :, 0]) plot_rgb_image(x[0], y[0, :, :, 0]) # define model, load weights, compile model model = pspnet(input_shape=(inputHeight_, inputWidth_, inputChannel_), n_class=numClass_, one_hot=oneHot_, backbone_name=backboneName_, backbone_weights=backboneWeights_, output_stride=8) if os.path.exists(resumeModelPath_): log('load weights from `{}`'.format(resumeModelPath_)) model.load_weights(resumeModelPath_, by_name=True, skip_mismatch=True) else: log('build new model `{}`'.format(saveModelPath_)) plot_model(model, to_file=plotPath_) with open(plotPath_.replace('png', 'json'), 'w') as f: f.write(model.to_json()) if modelSummary_: model.summary() log('model input: {}, output: {}'.format(model.input_shape, model.output_shape)) log('trainable parameters: {}'.format( int( np.sum([ K.count_params(p) for p in set(model.trainable_weights) ])))) log('non-trainable parameters: {}'.format( int( np.sum([ K.count_params(p) for p in set(model.non_trainable_weights) ])))) model.compile(optimizer=optimizer_, loss=loss_, metrics=['acc'] + metrics_) # train model.fit_generator(generator=train_generator, steps_per_epoch=stepsPerEpoch_, validation_data=val_generator, validation_steps=stepsPerValEpoch_, epochs=epoch_, callbacks=callbacks_, verbose=verbose_, initial_epoch=0) model.save_weights(filepath=checkpointDir_ + '/{}_{}.h5'.format(saveVersion_, epoch_)) log('training success!')
def main(): start = time.time() gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: try: # Currently, memory growth needs to be the same across GPUs for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) logical_gpus = tf.config.experimental.list_logical_devices('GPU') print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") except RuntimeError as e: # Memory growth must be set before GPUs have been initialized print(e) epochs = 30 image_size = 299 batch_size = 10 weights = True version = 4 weights_save_name = "EfficientNetB7_w_NT_" + str( image_size) + "_batch_" + str(batch_size) + "_v{}".format(str(version)) # Training dataset loading train_data = np.load("./dataset_npy/train_data_NT_fpv_25_rev_299.npy") train_label = np.load("./dataset_npy/train_label_NT_fpv_25_rev_299.npy") print("Dataset Loaded...") # Train and validation split trainX, valX, trainY, valY = train_test_split(train_data, train_label, test_size=0.2, shuffle=True) print(trainX.shape, valX.shape, trainY.shape, valY.shape) # Train nad validation image data generator trainAug = ImageDataGenerator( rescale=1.0 / 255.0, rotation_range=30, zoom_range=0.15, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15, horizontal_flip=True, fill_mode="nearest", ) valAug = ImageDataGenerator(rescale=1.0 / 255.0) model = cnn_model(img_size=image_size, weights=weights) # Number of trainable and non-trainable parameters trainable_count = int( np.sum([K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int( np.sum([K.count_params(p) for p in set(model.non_trainable_weights)])) print("Total params: {:,}".format(trainable_count + non_trainable_count)) print("Trainable params: {:,}".format(trainable_count)) print("Non-trainable params: {:,}".format(non_trainable_count)) if not exists("./models"): makedirs("./models") if not exists("./training_logs"): makedirs("./training_logs") if not exists("./plots"): makedirs("./plots") # Keras backend model_checkpoint = ModelCheckpoint( "models/" + weights_save_name + ".hdf5", monitor="val_loss", verbose=1, save_best_only=True, save_weights_only=True, ) stopping = EarlyStopping(monitor="val_loss", patience=5, verbose=1, mode='min') csv_logger = CSVLogger( "training_logs/EfficientNetB7_NT_{}_batch_{}_v{}.log".format( str(image_size), str(batch_size), str(version)), separator=",", append=True, ) print("Training is going to start in 3... 2... 1... ") # Model Training H = model.fit_generator( trainAug.flow(trainX, trainY, batch_size=batch_size), steps_per_epoch=len(trainX) // batch_size, validation_data=valAug.flow(valX, valY), validation_steps=len(valX) // batch_size, epochs=epochs, callbacks=[model_checkpoint, stopping, csv_logger], ) ''' H = model.fit( trainX, trainY, batch_size=batch_size, #steps_per_epoch=len(trainX) // batch_size, #validation_data=valAug.flow(valX, valY), validation_data=(valX, valY), #validation_steps=len(valX) // batch_size, epochs=epochs, callbacks=[model_checkpoint, stopping, csv_logger], ) ''' # plot the training loss and accuracy plt.style.use("ggplot") plt.figure() N = stopping.stopped_epoch + 1 plt.plot(np.arange(0, N), H.history["loss"], label="train_loss") plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss") plt.plot(np.arange(0, N), H.history["accuracy"], label="train_acc") plt.plot(np.arange(0, N), H.history["val_accuracy"], label="val_acc") plt.title("Training Loss and Accuracy") plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend(loc="lower left") plt.savefig( "plots/EfficientNetB7_training_plot_NT_{}_batch_{}_v{}.png".format( str(image_size), str(batch_size), str(version))) end = time.time() dur = end - start if dur < 60: print("Execution Time:", dur, "seconds") elif dur > 60 and dur < 3600: dur = dur / 60 print("Execution Time:", dur, "minutes") else: dur = dur / (60 * 60) print("Execution Time:", dur, "hours")