model = Model(inputs=base_model.input, outputs=predictions) return model model = get_pretrained() model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) #%% Training and logging from tensorflow.keras.callbacks import CSVLogger logdir = 'C:\\Users\\Mikko Impiö\\Google Drive\\koulu_honmia\\kandi19\\logs' csv_logger = CSVLogger(os.path.join(logdir, '09-01-2020_2.log'), append=True) tr_steps = len(df_train) // BATCH_SIZE val_steps = len(df_val) // BATCH_SIZE model.fit_generator(train_ds, validation_data=val_ds, steps_per_epoch=tr_steps, epochs=2, validation_steps=val_steps, callbacks=[csv_logger]) #%% Inference test_ds = test_ds.batch(BATCH_SIZE) model.evaluate(test_ds)
dtw_clusters=0, file_prefix="Oweights", skip_array=[], weight_pred_ind=False, weighs_dtw_cluster_ind=True) norm_model.compile( optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) norm_model_hist = norm_model.fit( train_images, train_labels, epochs=epochs, validation_data=(test_images, test_labels), callbacks=[CSVLogger(logs_norm, append=True), callback_weights_no_pred]) save_graph_plot(norm_model, project_paths["plots"] + "/norm_model.ps") save_graph_json(norm_model, project_paths["plots"] + "/norm_model.json") model_list.append(norm_model_hist) model_name_list.append("norm_model") logs_reg = project_paths["weights"] + "/" + "regModelE" + str( epochs) + "Skp" + str(total_skips) + ".csv" #Callbacks reg_train_steps = 3 clusters = 200 callback_weights_reg = StoreWeights(project_paths["weights"], reg_train_steps=reg_train_steps, dtw_clusters=clusters, file_prefix="Rweights",
color_mode="rgb", class_mode='binary') dev_generator = test_datagen.flow_from_directory('./dataset/dev', target_size=(target_side_len, target_side_len), batch_size=batch_size, color_mode="rgb", class_mode='binary') initial_model = VGG16(include_top=False, weights='imagenet', pooling='max') input = Input(shape=(224, 224, 3), name='image_input') x = Flatten()(initial_model(input)) x = Dense(200, activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.5)(x) x = Dense(1)(x) model = Model(inputs=input, outputs=x) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', f1]) csv_logger = CSVLogger('VGG16_results.csv', append=True, separator=';') model.fit_generator(train_generator, steps_per_epoch=2000, epochs=epochs, callbacks=[csv_logger], validation_data=dev_generator, validation_steps=800)
def train_model(self, runName, path_to_train_mix, path_to_train_speech, \ path_to_val_mix, path_to_val_speech): ''' Method to train the DTLN model. ''' # create save path if not existent savePath = './models_'+ runName+'/' if not os.path.exists(savePath): os.makedirs(savePath) # create log file writer csv_logger = CSVLogger(savePath+ 'training_' +runName+ '.log') # create callback for the adaptive learning rate reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=10**(-10), cooldown=1) # create callback for early stopping early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='auto', baseline=None) # create model check pointer to save the best model checkpointer = ModelCheckpoint(savePath+runName+'.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode='auto', save_freq='epoch' ) # calculate length of audio chunks in samples len_in_samples = int(np.fix(self.fs * self.len_samples / self.block_shift)*self.block_shift) # create data generator for training data generator_input = audio_generator(path_to_train_mix, path_to_train_speech, len_in_samples, self.fs, train_flag=True) dataset = generator_input.tf_data_set dataset = dataset.batch(self.batchsize, drop_remainder=True).repeat() # calculate number of training steps in one epoch steps_train = generator_input.total_samples//self.batchsize # create data generator for validation data generator_val = audio_generator(path_to_val_mix, path_to_val_speech, len_in_samples, self.fs) dataset_val = generator_val.tf_data_set dataset_val = dataset_val.batch(self.batchsize, drop_remainder=True).repeat() # calculate number of validation steps steps_val = generator_val.total_samples//self.batchsize # start the training of the model self.model.fit( x=dataset, batch_size=None, steps_per_epoch=steps_train, epochs=self.max_epochs, verbose=1, validation_data=dataset_val, validation_steps=steps_val, callbacks=[checkpointer, reduce_lr, csv_logger, early_stopping], max_queue_size=50, workers=4, use_multiprocessing=True) # clear out garbage tf.keras.backend.clear_session()
tf.keras.layers.GlobalAveragePooling2D( activity_regularizer=tf.keras.regularizers.l2(0.001)), tf.keras.layers.Dense(len(types), activation='sigmoid') ]) # plot_model(model1, to_file='model_complex.png', show_shapes=True) # exit() model1.compile( optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=['acc', f1_m, precision_m, recall_m, tf.keras.metrics.AUC()]) date = datetime.now().strftime("_%m_%d_%Y_%H_%M_%S") csv_logger = CSVLogger('logs/log_all' + date + '.csv') early_stop = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=5, mode='min', verbose=1, restore_best_weights=True) model_path = 'saved_models/best_model_all' + date + '.h5' mc = ModelCheckpoint(model_path, monitor='val_loss', mode='min', verbose=1) history = model1.fit_generator(train_generator, epochs=25, steps_per_epoch=steps_train, validation_data=valid_generator, validation_steps=steps_valid, verbose=2, callbacks=[csv_logger, mc, early_stop])
model.compile( loss="binary_crossentropy", optimizer=tf.keras.optimizers.Adam(lr), metrics=[ tf.keras.metrics.MeanIoU(num_classes=2), tf.keras.metrics.Recall(), tf.keras.metrics.Precision() ] ) # model.summary() callbacks = [ ModelCheckpoint(model_path, monitor="val_loss", verbose=1), ReduceLROnPlateau(monitor="val_loss", patience=5, factor=0.1, verbose=1), CSVLogger(csv_path), EarlyStopping(monitor="val_loss", patience=10) ] train_steps = len(train_x)//batch_size if len(train_x) % batch_size != 0: train_steps += 1 test_steps = len(test_x)//batch_size if len(test_x) % batch_size != 0: test_steps += 1 model.fit( train_dataset, validation_data=test_dataset, epochs=epochs,
def save_history(history): with open('history.db', 'wb') as file_pi: pickle.dump(history.history, file_pi) if __name__ == "__main__": epochs = 50 data_loader = load_data( "/home/ctadmin/data_drive/kits19/data/training_patches") validation_loader = load_data("/home/kits/kits19/data/validation_patches", is_validation=True) callbacks = list() callbacks.append( ModelCheckpoint("best_VNetW.h5", monitor='val_loss', save_weights_only=True, save_best_only=True, verbose=1)) # callbacks.append(ReduceLROnPlateau(factor=0.5, patience=2, verbose=1)) callbacks.append(EarlyStopping(monitor='val_loss', patience=10)) callbacks.append(CSVLogger("training.log", append=True)) model = unet() history = model.fit_generator(data_loader, validation_data=validation_loader, epochs=epochs, callbacks=callbacks) model.save_weights("weights/VNetW.h5") save_history(history)
def trainModel(epochs, bn_layers, dropout_layers, l2_layers, padding, target_size, dense_sizes, architecture, conv_layers_over_5, use_maxpool_after_conv_layers_after_5th, version, load_existing, gpu_id, model_filename, lc_filename, data_dir): # Trains a model # model = optional parameter; creates new if not passed; otherwise keeps training # epochs - number of max epochs to train (subject to early stopping) # bn_layers - list of indexes of Dense layers (-1 and down) and CNN layers (1 and up) where Batch Norm should be applied # dropout_layers - list of indexes of Dense layers (-1 and down) where Dropout should be applied # bn_layers - list of indexes of Dense layers (-1 and down) where L2 regularization should be applied # padding - changed to "same" to keep 2^n feature map sizes # dense_sizes - dictionary of dense layer sizes (cnt of neurons) # architecture - one of: Model_6classes_c4_d3_v1, Model_6classes_c5_d2_v1, Model_6classes_c5_d3_v1 # conv_layers_over_5 - number of convolutional layers after 5th # use_maxpool_after_conv_layers_after_5th - list of boolean values whether to use maxpooling after 5th layer # version - used to name a learning curve file # load_existing - whether to load an existing model file # Returns: # model: trained Keras model # # To call: # model = Train_v1.trainModel(epochs=20) crop_range = 1 # number of pixels to crop image (if size is 235, crops are 0-223, 1-224, ... 11-234) #target_size = 224 batch_size = 32 #datasrc = "visible" # Manually copied to C: to speed up training #data_dir = os.path.join(Glb.images_folder, "Bal_v14", "Ind-{}".format(hier_lvl) ) data_dir_train = os.path.join(data_dir, "Train") data_dir_val = os.path.join(data_dir, "Val") data_dir_test = os.path.join(data_dir, "Test") train_iterator = Glb_Iterators.get_iterator(data_dir_train, "div255") val_iterator = Glb_Iterators.get_iterator(data_dir_val, "div255") test_iterator = Glb_Iterators.get_iterator( data_dir_test, "div255", shuffle=False ) # dont shuffle in order to get proper actual/prediction pairs Softmax_size = len(train_iterator.class_indices) dense_sizes["d-1"] = Softmax_size #model_filename = os.path.join(Glb.results_folder, # "model_clsf_from_isVisible_{}_gpu{}_hier{}.h5".format(date.today().strftime("%Y%m%d"), gpu_id, hier_lvl)) #lc_filename = os.path.join(Glb.results_folder, # "lc_clsf_from_isVisible_{}_gpu{}_hier{}.csv".format(date.today().strftime("%Y%m%d"), gpu_id, hier_lvl)) # Create or load model if not load_existing: print("Creating model") prepModel = modelVersions_dic[architecture] prep_model_params = { "input_shape": (target_size, target_size, 3), "bn_layers": bn_layers, "dropout_layers": dropout_layers, "l2_layers": l2_layers, "padding": padding, "dense_sizes": dense_sizes, "conv_layers_over_5": conv_layers_over_5, "use_maxpool_after_conv_layers_after_5th": use_maxpool_after_conv_layers_after_5th } model = prepModel(**prep_model_params) else: print("Loading model") #model_filename = r"J:\Visible_models\6class\model_6classes_v" + str(version) + ".h5" model = load_model(model_filename) model.compile( loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001), # default LR: 0.001 metrics=['accuracy']) print(model.summary()) callback_earlystop = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=10, verbose=1, mode='max', restore_best_weights=True) callback_csv_logger = CSVLogger(lc_filename, separator=",", append=False) mcp_save = ModelCheckpoint(model_filename, save_best_only=True, monitor='val_accuracy', mode='max') model.fit(train_iterator, steps_per_epoch=len(train_iterator), epochs=epochs, verbose=2, validation_data=val_iterator, validation_steps=len(val_iterator), callbacks=[callback_csv_logger, callback_earlystop, mcp_save]) # print("Evaluation on test set (1 frame)") test_metrics = model.evaluate(test_iterator) print("Test: {}".format(test_metrics)) print("Evaluating F1 test set (1 frame)") y_pred = model.predict(test_iterator) y_pred_classes = np.argmax(y_pred, axis=1) y_true = test_iterator.classes test_acc = accuracy_score(y_true=y_true, y_pred=y_pred_classes) test_f1 = f1_score(y_true=y_true, y_pred=y_pred_classes, average='macro') print("acc:{}, f1:{}".format(test_acc, test_f1)) # metrics to csv df_metrics = pd.DataFrame( data={ "gpu": [gpu_id], "datetime": [datetime.now().strftime("%Y%m%d %H:%M:%S")], "data_dir": [data_dir], "test_acc": [test_acc], "test_f1": [test_f1] }) df_metrics_filename = os.path.join(Glb.results_folder, "metrics_mrg.csv") df_metrics.to_csv(df_metrics_filename, index=False, header=False, mode='a') #print("Evaluation on validation set (1 frame)") #val_metrics = model.evaluate(val_iterator) #print("Val: {}".format(val_metrics)) return model
def main(): good_path = "/home/osboxes/DeepLearningResearch/Data/badging_med/mal_badging_med.txt" mal_path = "/home/osboxes/DeepLearningResearch/Data/badging_med/ben_badging_med.txt" tr = .80 batch = 100 epochs = 10 neurons = 20 perm_inputs, feat_inputs, labels = vectorize(good_path, mal_path) print("returned from vectorize method" + str(perm_inputs) + "feat Inputs" + str(feat_inputs) + "labels" + str(labels)) perm_width = int(len(perm_inputs[0])) print("perm_width: " + str(perm_width)) feat_width = int(len(feat_inputs[0])) print("feat_width: " + str(feat_width)) cm = np.zeros([2, 2], dtype=np.int64) #neurons = [10, 20, 30, 40] optimizers = ['nadam', 'adam', 'RMSprop', 'SGD'] #for neuronVar in neurons: for optimizerVar in optimizers: if os.path.exists('log_' + optimizerVar + '.csv'): os.remove('log_' + optimizerVar + '.csv') else: print("The file does not exist") print("OPTIMIZER: " + str(optimizerVar)) model = create_dualInputLarge(input_ratio=.125, neurons=20, perm_width=perm_width, feat_width=feat_width, optimizer=optimizerVar) plot_model(model, to_file='model.png') # model.summary() time.sleep(1) sss = StratifiedShuffleSplit(n_splits=1, random_state=0, test_size=1 - tr) i = 0 print("stratified shuffle split") for train_index, test_index in sss.split(perm_inputs, labels): perm_train, perm_test = perm_inputs[train_index], perm_inputs[ test_index] feat_train, feat_test = feat_inputs[train_index], feat_inputs[ test_index] labels_train, labels_test = labels[train_index], labels[test_index] #print ("perm_width: " + str(perm_width)) #print ("feat_width: " + str(feat_width)) model = create_dualInputLarge(input_ratio=.125, neurons=20, perm_width=perm_width, feat_width=feat_width, optimizer=optimizerVar) print('\nsplit %i' % i) csv_logger = CSVLogger('log_' + optimizerVar + '.csv', append=True, separator=',') model.fit([perm_train, feat_train], labels_train, epochs=epochs, batch_size=batch, callbacks=[csv_logger]) print("model trained") labels_pred = model.predict([perm_test, feat_test], batch_size=batch) # print("prediction made: " +str(labels_pred)) labels_pred = (labels_pred > 0.5) #print("labels_pred" +str(labels_pred)) cm = cm + confusion_matrix(labels_test, labels_pred) i += 1 acc = calc_accuracy(cm) print('average accuracy was: ' + str(acc)) precision = calc_precision(cm) print('Average precision was: ' + str(precision)) recall = cal_recall(cm) print('Average recall value is: ' + str(recall)) #scoring = ['precision', 'accuracy', 'recall', 'f1'] #print("creating the loaded model") # loaded_model = KerasClassifier(build_fn=get_model(model), epochs=epochs, batch_size=batch, verbose=2) # print("calling the cross_validate method") # fit_params = dict(batch_size=batch, epochs=epochs) #cv_result = cross_validate(loaded_model, perm_inputs, labels, fit_params=fit_params, cv=sss, return_train_score=True, n_jobs=1, verbose=2) #df = pandas.DataFrame(cv_result) #path1 = '/home/osboxes/DeepLearningResearch/Demo/test' + '.csv' # file1 = open(path1, "a+") #df.to_csv(file1, index=True) # file1.close() return
def fit_with_generator( self, train_data_raw: np.ndarray, labels_raw: np.ndarray, model_filepath: str, weights_filepath: str, logs_dir: str, training_log: str, resume: bool, ) -> Tuple[List[float], List[float]]: """Fit the training data to the network and save the network model as a HDF file. Arguments: train_data_raw {list} -- The HDF5 raw training data. labels_raw {list} -- The HDF5 raw training labels. model_filepath {string} -- The model file path. weights_filepath {string} -- The weights file path. logs_dir {string} -- The TensorBoard log file directory. training_log {string} -- The path to the log file of epoch results. resume {bool} -- True to continue with previous training result or False to start a new one (default: {False}). Returns: tuple -- A tuple contains validation losses and validation accuracies. """ initial_epoch = 0 batch_size = self.hyperparameters.batch_size validation_split = self.hyperparameters.validation_split csv_logger = (CSVLogger(training_log) if not resume else CSVLogger(training_log, append=True)) checkpoint = ModelCheckpoint( filepath=weights_filepath, monitor=self.hyperparameters.monitor, verbose=1, save_best_only=False, save_weights_only=True, ) tensorboard = TensorBoard( log_dir=logs_dir, histogram_freq=0, write_graph=True, write_images=True, ) earlyStopping = EarlyStopping( monitor=self.hyperparameters.monitor, min_delta=self.hyperparameters.es_min_delta, mode=self.hyperparameters.es_mode, patience=self.hyperparameters.es_patience, verbose=1) callbacks_list = [ checkpoint, tensorboard, csv_logger, earlyStopping, ] if not resume: Optimizer = getattr(tf_optimizers, self.hyperparameters.optimizer) self.__model.compile( loss=self.hyperparameters.loss, optimizer=Optimizer( learning_rate=self.hyperparameters.learning_rate), metrics=self.hyperparameters.metrics, ) if resume: assert os.path.isfile( training_log ), "{} does not exist and is required by training resumption".format( training_log) training_log_file = open(training_log) initial_epoch += sum(1 for _ in training_log_file) - 1 training_log_file.close() assert self.hyperparameters.epochs > initial_epoch, \ "The existing model has been trained for {0} epochs. Make sure the total epochs are larger than {0}".format(initial_epoch) train_generator = self.__generator(train_data_raw, labels_raw, batch_size, validation_split, is_validation=False) test_generator = self.__generator(train_data_raw, labels_raw, batch_size, validation_split, is_validation=True) steps_per_epoch = math.ceil( float(train_data_raw.shape[0]) * (1 - validation_split) / batch_size) validation_steps = math.ceil( float(train_data_raw.shape[0]) * validation_split / batch_size) try: hist = self.__model.fit( train_generator, steps_per_epoch=steps_per_epoch, validation_data=test_generator, validation_steps=validation_steps, epochs=self.hyperparameters.epochs, shuffle=False, callbacks=callbacks_list, initial_epoch=initial_epoch, ) except KeyboardInterrupt: Network.__LOGGER.warning("Training interrupted by the user") raise TerminalException("Training interrupted by the user") finally: self.__model.save(model_filepath) Network.__LOGGER.warning("Model saved to %s" % model_filepath) return hist.history["val_loss"], hist.history["val_acc"] if int( tf.__version__.split(".")[0]) < 2 else hist.history["val_accuracy"]
model.add(Conv2D(64, (3, 3), strides = 1, activation = 'relu', kernel_initializer = 'he_uniform')) model.add(MaxPooling2D((2, 2), strides = (2, 2), padding = 'same')) model.add(Flatten()) model.add(Dense(100, activation = 'relu', kernel_initializer = 'he_uniform')) model.add(Dense(10, activation = 'softmax')) model.compile(optimizer = SGD(learning_rate = 0.01, momentum = 0.9), loss = 'categorical_crossentropy', metrics = ['accuracy']) if not os.path.isdir('Model'): os.mkdir('Model') callbacks = [ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 7, min_lr = 1e-5), EarlyStopping(patience = 9, # Patience should be larger than the one in ReduceLROnPlateau min_delta = 1e-5), CSVLogger(os.path.join('Model', 'training.log'), append = True), ModelCheckpoint(os.path.join('Model', 'backup_last_model.hdf5')), ModelCheckpoint(os.path.join('Model', 'best_val_acc.hdf5'), monitor = 'val_accuracy', mode = 'max', save_best_only = True), ModelCheckpoint(os.path.join('Model', 'best_val_loss.hdf5'), monitor = 'val_loss', mode = 'min', save_best_only = True)] model.fit(trainGenerator, epochs = 50, validation_data = validationGenerator, callbacks = callbacks) model = load_model(os.path.join('Model', 'best_val_loss.hdf5')) loss, acc = model.evaluate(validationGenerator) print('Loss on Validation Data : ', loss) print('Accuracy on Validation Data :', '{:.4%}'.format(acc))
newModel.compile( loss="categorical_crossentropy", #keras.losses.binary_crossentropy optimizer=model_opt, metrics=['accuracy']) newModel.summary() if int(file_index) == 2: newModel = load_model("./Models/2CNN_1n1c1c_1.h5") ###################################################################################### # time counter print(time.strftime("%a %b %d %H:%M:%S %Y", time.localtime())) ticks_1 = time.time() ############################################################################################################################################################ check_list = [] csv_logger = CSVLogger("./Models/training_log_1n1c1c_" + str(file_index) + ".csv") checkpoint = ModelCheckpoint(filepath="./Models/checkmodel_1n1c1c_" + str(file_index) + ".h5", save_best_only=True, verbose=1) earlystopping = EarlyStopping( monitor="loss", min_delta=0.01, patience=50, verbose=1, mode="auto", baseline=None, restore_best_weights=False, ) check_list.append(checkpoint) check_list.append(csv_logger)
from hessianlearn import * # Memory issue with GPUs gpu_devices = tf.config.experimental.list_physical_devices('GPU') for device in gpu_devices: tf.config.experimental.set_memory_growth(device, True) import numpy as np import resnet_hl lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) early_stopper = EarlyStopping(min_delta=0.001, patience=10) csv_logger = CSVLogger('resnet18_cifar10.csv') batch_size = 32 nb_classes = 10 nb_epoch = 200 data_augmentation = True # input image dimensions img_rows, img_cols = 32, 32 # The CIFAR10 images are RGB. img_channels = 3 # The data, shuffled and split between train and test sets: (X_train, y_train), (X_test, y_test) = cifar10.load_data() # Convert class vectors to binary class matrices.
else: get_loss = {'dice': DiceLoss, 'jaccard': JaccardLoss, 'focal': FocalLoss} loss = get_loss[args.loss]() # compiling model model.compile(loss=loss, optimizer=Adam(), metrics=['mean_squared_error']) # creating directory for experiment callbacks = [] experiment_label = '_'.join([args.dataset, model.name, args.run_label]) experiment_path = os.path.join(args.save_path, experiment_label) if not os.path.exists(experiment_path): os.makedirs(experiment_path) # setting additional callbacks log = CSVLogger(os.path.join(experiment_path, 'optimization.log')) stop = EarlyStopping(patience=args.stop_patience) plateau = ReduceLROnPlateau(patience=args.reduce_patience) save_filename = os.path.join(experiment_path, 'model.hdf5') save = ModelCheckpoint(save_filename, save_best_only=True) callbacks.extend([log, stop, save, plateau]) # saving hyper-parameters and model summary with open(os.path.join(experiment_path, 'hyperparameters.json'), 'w') as filer: json.dump(args.__dict__, filer, indent=4) with open(os.path.join(experiment_path, 'model_summary.txt'), 'w') as filer: model.summary(print_fn=lambda x: filer.write(x + '\n')) # starting optimization model.fit(sequencers['train'], epochs=args.epochs,
############################################################# # Run training/testing ############################################################# # save weights when loss is improved checkpoint = ModelCheckpoint('{epoch:04d}-{val_loss:.4f}.hdf5', monitor='val_loss', verbose=0, save_best_only=True) # reduce learning rate after 2000 epochs learning_rate_reduce = LearningRateScheduler(learningRateReduce, verbose=0) # record logs after every epoch today = str(date.today()) csv_logger = CSVLogger('training-' + today + '.log') if TESTING: # record test loss (MAE) [loss, acc] = model.evaluate(DataGenerator(input_fns, input_gt_fns)) print('Loss: ' + str(loss)) # predict output images on test set out = model.predict(DataGenerator(input_fns, input_gt_fns), verbose=1) psnr = [] ssim = [] # record metrics and also save images to disk for idx, im in enumerate(out):
model.add(Activation('relu')) # Add Dropout model.add(Dropout(0.4)) # Output Layer model.add(Dense(2)) model.add(Activation('softmax')) model.summary() # Compile the model model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer='adam', metrics=['accuracy']) csv_logger = CSVLogger("metric_calc_5.csv", separator=",", append=True) model_history = model.fit(x=input_x, y=output_y, epochs=50, validation_data=(test_x, test_y), verbose=2, callbacks=[csv_logger]) model.save("alexnet_v4_calc.h5") calc_metrics = pd.DataFrame(columns=["loss", "acc", "val_loss", "val_acc"]) calc_metrics = calc_metrics.append(pd.DataFrame(model_history.history), ignore_index=True) calc_metrics.to_csv(r"calc_metrics_6_pd_to_csv.csv") ##y_pred = model.predict(input_x) ##y_pred = y_pred.ravel() #flatten both pred and true y
def main(go_file, train_data_file, test_data_file, terms_file, model_file, out_file, split, batch_size, epochs, load, logger_file, threshold, device, params_index): params = { 'max_kernel': 129, 'initializer': 'glorot_normal', 'dense_depth': 0, 'nb_filters': 512, 'optimizer': Adam(lr=3e-4), 'loss': 'binary_crossentropy' } # SLURM JOB ARRAY INDEX pi = params_index if params_index != -1: kernels = [33, 65, 129, 257, 513] dense_depths = [0, 1, 2] nb_filters = [32, 64, 128, 256, 512] params['max_kernel'] = kernels[pi % 5] pi //= 5 params['dense_depth'] = dense_depths[pi % 3] pi //= 3 params['nb_filters'] = nb_filters[pi % 5] pi //= 5 out_file = f'data/predictions_{params_index}.pkl' logger_file = f'data/training_{params_index}.csv' model_file = f'data/model_{params_index}.h5' print('Params:', params) go = Ontology(go_file, with_rels=True) terms_df = pd.read_pickle(terms_file) terms = terms_df['terms'].values.flatten() train_df, valid_df = load_data(train_data_file, terms, split) test_df = pd.read_pickle(test_data_file) terms_dict = {v: i for i, v in enumerate(terms)} nb_classes = len(terms) with tf.device('/' + device): test_steps = int(math.ceil(len(test_df) / batch_size)) test_generator = DFGenerator(test_df, terms_dict, nb_classes, batch_size) if load: logging.info('Loading pretrained model') model = load_model(model_file) else: logging.info('Creating a new model') model = create_model(nb_classes, params) logging.info("Training data size: %d" % len(train_df)) logging.info("Validation data size: %d" % len(valid_df)) checkpointer = ModelCheckpoint(filepath=model_file, verbose=1, save_best_only=True) earlystopper = EarlyStopping(monitor='val_loss', patience=6, verbose=1) logger = CSVLogger(logger_file) logging.info('Starting training the model') valid_steps = int(math.ceil(len(valid_df) / batch_size)) train_steps = int(math.ceil(len(train_df) / batch_size)) train_generator = DFGenerator(train_df, terms_dict, nb_classes, batch_size) valid_generator = DFGenerator(valid_df, terms_dict, nb_classes, batch_size) model.summary() model.fit_generator(train_generator, steps_per_epoch=train_steps, epochs=epochs, validation_data=valid_generator, validation_steps=valid_steps, max_queue_size=batch_size, workers=12, callbacks=[logger, checkpointer, earlystopper]) logging.info('Loading best model') model = load_model(model_file) logging.info('Evaluating model') loss = model.evaluate_generator(test_generator, steps=test_steps) logging.info('Test loss %f' % loss) logging.info('Predicting') test_generator.reset() preds = model.predict_generator(test_generator, steps=test_steps) # valid_steps = int(math.ceil(len(valid_df) / batch_size)) # valid_generator = DFGenerator(valid_df, terms_dict, # nb_classes, batch_size) # logging.info('Predicting') # valid_generator.reset() # preds = model.predict_generator(valid_generator, steps=valid_steps) # valid_df.reset_index() # valid_df['preds'] = list(preds) # train_df.to_pickle('data/train_data_train.pkl') # valid_df.to_pickle('data/train_data_valid.pkl') test_labels = np.zeros((len(test_df), nb_classes), dtype=np.int32) for i, row in enumerate(test_df.itertuples()): for go_id in row.prop_annotations: if go_id in terms_dict: test_labels[i, terms_dict[go_id]] = 1 logging.info('Computing performance:') roc_auc = compute_roc(test_labels, preds) logging.info('ROC AUC: %.2f' % (roc_auc, )) test_df['labels'] = list(test_labels) test_df['preds'] = list(preds) logging.info('Saving predictions') test_df.to_pickle(out_file)
plot_model(model, 'model.jpg') if training: checkpoint = ModelCheckpoint(model_path, 'val_class_acc', verbose=1, save_best_only=True, save_weights_only=True) reduce_lr = ReduceLROnPlateau('val_class_acc', 0.5, 10, verbose=1, min_lr=1e-6) early_stopping = EarlyStopping('val_class_acc', patience=50, restore_best_weights=True) logger = CSVLogger(model_path + '.csv', append=True) tensorboard = TensorBoard(model_path[:model_path.rfind('.')] + '_logs', batch_size=1024, update_freq='epoch') #model.fit(trainX, trainY, batch_size=128, epochs=500, validation_data=(validX, validY), verbose=2, callbacks=[checkpoint, reduce_lr, early_stopping, logger, tensorboard]) model.fit( trainX, [trainY, missing_col], batch_size=128, epochs=500, validation_data=(validX, [validY, valid_missing_col]), verbose=2, callbacks=[checkpoint, reduce_lr, early_stopping, logger, tensorboard]) if submit: if svm: testX = utils.load_test_data(submit)
# all but last 5 layers for layer in model.layers[:-5]: layer.trainable = False # last 5 layers for layer in model.layers[-5:]: layer.trainable = True model.compile(optimizer=SGD(learning_rate=0.01, momentum=0.9), loss="categorical_crossentropy", metrics=["accuracy"]) model.summary() output_folder_path = os.path.join("hyperparameter-tuning-mobilenetv2", set_folder) if not os.path.exists(output_folder_path): os.makedirs(output_folder_path) csv_logger_filepath = os.path.join(output_folder_path, "history.log") csv_logger = CSVLogger(csv_logger_filepath) history = model.fit(train_generator, epochs=num_epochs, validation_data=val_generator, verbose=1, callbacks=[csv_logger] ) # probably won't reach here becuase training would be stopped in between history_object_filepath = os.path.join(output_folder_path, "history_dict") with open(history_object_filepath, "wb") as history_object: pickle.dump(history.history, history_object)
def train(): max_features = 10000 sequence_length = 250 # Load all datasets raw_train_ds = load_train_dataset() raw_val_ds = load_val_dataset() raw_test_ds = load_test_dataset() # Define the vectorization layer global vectorization_layer vectorization_layer = TextVectorization( standardize='lower_and_strip_punctuation', max_tokens=max_features, output_mode='int', output_sequence_length=sequence_length) # Adapt the vectorization layer vectorization_layer.adapt(raw_train_ds.map(lambda x, y: x)) # Vectorize the dataset train_ds = raw_train_ds.map(text_to_vector) val_ds = raw_val_ds.map(text_to_vector) test_ds = raw_test_ds.map(text_to_vector) auto_tune = tf.data.experimental.AUTOTUNE train_ds = train_ds.cache().prefetch(buffer_size=auto_tune) val_ds = val_ds.cache().prefetch(buffer_size=auto_tune) test_ds = test_ds.cache().prefetch(buffer_size=auto_tune) embedding_dim = 32 # Define the model model = tf.keras.Sequential([ layers.Embedding(max_features + 1, embedding_dim), layers.Dropout(0.2), layers.GlobalAveragePooling1D(), layers.Dropout(0.2), layers.Dense(embedding_dim, activation='relu'), layers.Dropout(0.2), layers.Dense(1) ]) # Compile the model model.compile(loss=losses.BinaryCrossentropy(from_logits=True), optimizer='adam', metrics=tf.metrics.BinaryAccuracy(threshold=0.0)) # Train the model epochs = 10 csv_logger = CSVLogger('training.log') history = model.fit(train_ds, validation_data=val_ds, epochs=epochs, callbacks=[csv_logger]) # Test the model loss, accuracy = model.evaluate(test_ds) print("Model loss against test dataset = ", loss) print("Model accuracy against test dataset = ", accuracy) # Save the model export_model = tf.keras.Sequential([ tf.keras.Input(shape=(1, ), dtype="string"), vectorization_layer, model, layers.Activation('sigmoid') ]) export_model.compile(loss=losses.BinaryCrossentropy(from_logits=False), optimizer="adam", metrics=['accuracy']) export_model.save('saved_model')
# with CustomObjectScope({'loss': sm.losses.bce_jaccard_loss, 'metrics': sm.metrics.iou_score}): model = load_model(model_path, custom_objects={ 'loss': loss, 'metrics': metrics }) else: model = sm.Unet('seresnet18', classes=1, activation='sigmoid') model.compile( 'Adam', loss=loss, metrics=metrics, ) csv_logger = CSVLogger( f"{file_path}{model_name}_{batch_size}_{epochs}.csv", append=False) checkpoint = ModelCheckpoint(model_path, verbose=1, save_best_only=True) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, min_lr=1e-6, verbose=1) early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=False) callbacks = [csv_logger, checkpoint, reduce_lr, early_stopping] model.fit_generator(train_gen, validation_data=valid_gen, steps_per_epoch=train_steps, validation_steps=valid_steps,
def callbacks_req(model_type='LSTM'): csv_logger = CSVLogger(model_folder+'/training-log-'+model_type+'-'+str(test_year)+'.csv') filepath = model_folder+"/model-" + model_type + '-' + str(test_year) + "-E{epoch:02d}.h5" model_checkpoint = ModelCheckpoint(filepath, monitor='val_loss',save_best_only=True) earlyStopping = EarlyStopping(monitor='val_loss',mode='min',patience=10,restore_best_weights=True) return [csv_logger,earlyStopping,model_checkpoint]
LRScheduler = LearningRateScheduler(lrSchedule) # Create checkpoint for the training # This checkpoint performs model saving when # an epoch gives highest testing accuracy filepath = modelname + ".hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max') # Log the epoch detail into csv csv_logger = CSVLogger(modelname + '.csv') callbacks_list = [checkpoint, csv_logger, LRScheduler] # Fit the model datagen = ImageDataGenerator(width_shift_range=0.2, height_shift_range=0.2, rotation_range=20, horizontal_flip=True, vertical_flip=False) model.fit_generator( datagen.flow(trDat, trLbl, batch_size=32), validation_data=(tsDat, tsLbl), epochs=200, #originally 200 verbose=1, steps_per_epoch=len(trDat) / 32,
def set_log_callback(self): '''Set a logger to write the accuracies and losses in each epic to a file''' csv_logger = CSVLogger(self.output_dir + '\\training.log') self.callbacks.append(csv_logger)
skip_array=skip_steps, weight_pred_ind=True, weighs_dtw_cluster_ind=True, replicate_csvs_at=0) checkpoint_path = project_paths["checkpoints"] + "/weights_epoch-{epoch}.ckpt" restore_path = project_paths["checkpoints"] + "/weights_epoch-" + str( FirstSkip) + ".ckpt" callback_save_model_reg = ModelCheckpoint(filepath=checkpoint_path, save_weights_only=False, verbose=1, period=FirstSkip) project_paths = get_project_paths(sys.argv[0], to_tmp=False) logs = project_paths["weights"] + "/reg_model_history_log.csv" csv_logger = CSVLogger(logs, append=True) (x_train, _), (x_test, _) = tf.keras.datasets.mnist.load_data() x_train = x_train.astype('float32') / 255. x_test = x_test.astype('float32') / 255. csv_logger = CSVLogger(logs, append=True) shutil.copy2(os.path.realpath(__file__), project_paths['code']) # copying code file to logs def create_model(): model = models.Sequential() model.add(layers.Flatten()) model.add(layers.Dense(latent_dim, activation='relu'))
model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model def get_layers(numbers): options = [] for a in numbers: for b in numbers: for c in numbers: options.append([a, b, c]) return options logger = CSVLogger('Ratio_logs.csv', append=True) sizes=[16, 32, 64, 128] models = [] for a, b, c in get_layers(sizes): model = create_model(a, b, c) models.append(model) best_loss = 100 for nn in models: print('_'*50) print(nn.summary()) nn.fit(X_train, y_train, callbacks=[logger], batch_size=32, epochs=100)
import numpy as np import random from dataloader import CaptchaSequence import string characters = string.digits + string.ascii_uppercase data = CaptchaSequence(characters, batch_size=2, steps=1000) from tensorflow.keras.models import * from tensorflow.keras.layers import * from tensorflow.keras.callbacks import EarlyStopping, CSVLogger, ModelCheckpoint from tensorflow.keras.optimizers import * train_data = CaptchaSequence(characters, batch_size=128, steps=1000) valid_data = CaptchaSequence(characters, batch_size=128, steps=100) callbacks = [ EarlyStopping(patience=3), CSVLogger('cnn.csv'), ModelCheckpoint('cnn_best.h5', save_best_only=True) ] from model import model model.compile(loss='categorical_crossentropy', optimizer=Adam(1e-3, amsgrad=True), metrics=['accuracy']) model.fit_generator(train_data, epochs=100, validation_data=valid_data, workers=4, use_multiprocessing=True, callbacks=callbacks)
def training(X_tr, Y_tr, X_v, Y_v): model = GRU_model() #compute weight, ideally, we don't need this if all classes are equal in number. weight = np.zeros(num_classes) # class #counting classes for i in Y_tr: weight[int(i)] += 1 print('Training class count') print(weight) d = np.min([1.0 / temp for temp in weight]) weight = [1.0 / temp / d for temp in weight] rmsprop = keras.optimizers.RMSprop(lr=0.003, rho=0.9, epsilon=1e-08, decay=0.0) adam = keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08) sgd = keras.optimizers.SGD(lr=0.01, momentum=0.0, decay=1e-6) # OBS rmsprop and adam cant get loss past 2, while sgd got as low as 0.2 model.compile(loss='sparse_categorical_crossentropy', optimizer=rmsprop, metrics=['sparse_categorical_accuracy']) reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=1500, min_lr=0.0001) csv_logger = CSVLogger('allnode_PIN.log') filepath = "allnode_weights-v-{epoch:02d}.hdf5" checkpointer = ModelCheckpoint(monitor='val_loss', filepath=filepath, verbose=1, save_best_only=True) start = time() model.fit_generator(generator(X_tr, Y_tr, 256), steps_per_epoch=4, epochs=epochs, validation_data=(X_v, Y_v), callbacks=[checkpointer, csv_logger, reduce_lr], max_queue_size=3, use_multiprocessing=True, class_weight=weight) end = time() print('time:', end - start) ''' open log file to find best weight ''' # open log file # log file contains loss of each epoch in each running step log_file = "./allnode_PIN.log" loss = [] with open(log_file) as f: f = f.readlines() f[0:] = f[1:] #delete header for line in f: loss.append([int(line.split(',')[0]), float(line.split(',')[3])]) #save all lost to list # find minimum loss # Keep minimum lost and save its index and use it for testing min_loss = 100 for data in range(len(loss)): if loss[data][1] < min_loss: min_loss = loss[data][1] best_model_index = loss[data][0] return best_model_index, end - start, loss[:][1]
def customTrainedModel( train_path, valid_path, model, optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'], filepath='saved_models/weights-improvement-{epoch:02d}.h5', monitor='val_accuracy', verbose=1, save_best_only=True, mode='max', logfile='my_logs.csv', separator=',', append=False, batch_size=DEFAULT_BATCH_SIZE, class_mode='categorical', epochs=DEFAULT_EPOCHS, target_size=TARGET_SIZE, featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, rotation_range=0, width_shift_range=CONSTANT_ZERO, height_shift_range=CONSTANT_ZERO, brightness_range=None, shear_range=CONSTANT_ZERO, zoom_range=CONSTANT_ZERO, channel_shift_range=CONSTANT_ZERO, fill_mode='nearest', cval=CONSTANT_ZERO, horizontal_flip=False, vertical_flip=False, rescale=None, preprocessing_function=None, data_format=None, validation_split=CONSTANT_ZERO, dtype=None, test_featurewise_center=False, test_samplewise_center=False, test_featurewise_std_normalization=False, test_samplewise_std_normalization=False, test_zca_whitening=False, test_zca_epsilon=1e-06, test_rotation_range=0, test_width_shift_range=CONSTANT_ZERO, test_height_shift_range=CONSTANT_ZERO, test_brightness_range=None, test_shear_range=CONSTANT_ZERO, test_zoom_range=CONSTANT_ZERO, test_channel_shift_range=CONSTANT_ZERO, test_fill_mode='nearest', test_cval=CONSTANT_ZERO, test_horizontal_flip=False, test_vertical_flip=False, test_rescale=None, test_preprocessing_function=None, test_data_format=None, test_validation_split=CONSTANT_ZERO, test_dtype=None, validation_freq=1, class_weight=None, max_queue_size=10, workers=1, use_multiprocessing=False, shuffle=True, initial_epoch=0): model = model model.compile(optimizer=optimizer, loss=loss, metrics=metrics) checkpoint = ModelCheckpoint(filepath=filepath, monitor=monitor, verbose=verbose, save_best_only=save_best_only, mode=mode) log_csv = CSVLogger(logfile, separator=separator, append=append) callable_list = [checkpoint, log_csv] train_datagen = ImageDataGenerator( featurewise_center=featurewise_center, samplewise_center=samplewise_center, featurewise_std_normalization=featurewise_std_normalization, samplewise_std_normalization=samplewise_std_normalization, zca_whitening=zca_whitening, zca_epsilon=zca_epsilon, rotation_range=rotation_range, width_shift_range=width_shift_range, height_shift_range=height_shift_range, brightness_range=brightness_range, shear_range=shear_range, zoom_range=zoom_range, channel_shift_range=channel_shift_range, fill_mode=fill_mode, cval=cval, horizontal_flip=horizontal_flip, vertical_flip=vertical_flip, rescale=rescale, preprocessing_function=preprocessing_function, data_format=data_format, validation_split=validation_split, dtype=dtype) test_datagen = ImageDataGenerator( featurewise_center=test_featurewise_center, samplewise_center=test_samplewise_center, featurewise_std_normalization=test_featurewise_std_normalization, samplewise_std_normalization=test_samplewise_std_normalization, zca_whitening=test_zca_whitening, zca_epsilon=test_zca_epsilon, rotation_range=test_rotation_range, width_shift_range=test_width_shift_range, height_shift_range=test_height_shift_range, brightness_range=test_brightness_range, shear_range=test_shear_range, zoom_range=test_zoom_range, channel_shift_range=test_channel_shift_range, fill_mode=test_fill_mode, cval=test_cval, horizontal_flip=test_horizontal_flip, vertical_flip=test_vertical_flip, rescale=test_rescale, preprocessing_function=test_preprocessing_function, data_format=test_data_format, validation_split=test_validation_split, dtype=test_dtype) training_set = train_datagen.flow_from_directory(train_path, target_size=target_size, batch_size=batch_size, class_mode=class_mode) test_set = test_datagen.flow_from_directory(valid_path, target_size=target_size, batch_size=batch_size, class_mode=class_mode) history = model.fit_generator(training_set, validation_data=test_set, epochs=epochs, steps_per_epoch=len(training_set), validation_steps=len(test_set), callbacks=callable_list, validation_freq=validation_freq, class_weight=class_weight, max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing, shuffle=shuffle, initial_epoch=initial_epoch) return history, model
model.add(BatchNormalization()) model.add(Dense(1)) model.add(Activation('sigmoid')) # In[9]: my_optimizer = optimizer_choice(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0, amsgrad=False) model.compile(optimizer=my_optimizer, loss='binary_crossentropy', metrics=['accuracy', f1, precision, recall]) # In[10]: csv_logger = CSVLogger('VGG16_results_new_nofreeze_5dropout.csv', append=True, separator=';') filepath="vgg_weights/VGG16_new_weights_5drop_regu-{epoch:02d}-{val_acc:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1) # In[11]: model.fit_generator( train_generator, steps_per_epoch= train_generator.samples//train_generator.batch_size, epochs=epochs, callbacks=[csv_logger], validation_data=dev_generator, validation_steps=dev_generator.samples//dev_generator.batch_size)