def get_model(max_words, maxlen, emb_dim=8, output=4): """ Returns multiclass classification model """ model = models.Sequential([ layers.Embedding(max_words, emb_dim, input_length=maxlen), layers.Flatten(), layers.Dense(output, activation='softmax') # 4 probability output ]) metrics = [ CategoricalAccuracy(), Precision(), Recall(), AUC() ] model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=metrics) return model
def init_metrics(paramDict): def jaccard_metric(): def fn(y_true, y_pred): return jaccard(y_true, y_pred) fn.__name__ = 'jaccard' return fn def dice_metric(): def fn(y_true, y_pred): return dice(y_true, y_pred) fn.__name__ = 'dice' return fn metrics = ['mse', CategoricalAccuracy(), jaccard_metric(), dice_metric()] return metrics
def __init__(self): self.model = Sequential() self.model.add(Flatten(input_shape=(32, 32, 3))) self.model.add(Dense(units=256, activation="relu")) self.model.add(Dropout(0.10)) self.model.add(Dense(units=128, activation="relu")) self.model.add(Dropout(0.10)) self.model.add(Dense(units=10, activation="softmax")) self.model.compile(optimizer=RMSprop(learning_rate=0.0001), loss=CategoricalCrossentropy(), metrics=[ Precision(name="precision"), Recall(name="recall"), CategoricalAccuracy(name="accuracy"), AUC(name="auc") ])
def RNN(x_train, y_train): model = Sequential() model.add( Embedding(max_features, 256, embeddings_initializer='lecun_normal')) model.add(SpatialDropout1D(rate=0.4)) model.add(Bidirectional(LSTM(units=64, dropout=0.4, recurrent_dropout=0.4))) model.add( Dense(units=y_train.shape[1], activation='softmax', kernel_initializer='lecun_normal')) METRICS = [ CategoricalAccuracy(name='accuracy'), Precision(name='precision'), Recall(name='recall'), AUC(name='auc') ] optimizer = Adam(lr=1e-5) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=METRICS) model.summary() earlystop = EarlyStopping(monitor='val_loss', patience=15, restore_best_weights=True) reduceLR = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(1e-1), verbose=1, patience=5) history = model.fit(x=x_train, y=y_train, batch_size=32, epochs=10000, verbose=0, callbacks=[earlystop, reduceLR], validation_split=0.3, shuffle=True, workers=4) return model
def do_training(initial_learning_rate=0.1): gids = get_gids_from_database("unet") training_gen, validation_gen = initialize_train_and_validation_generators( "unet", gids, batch_size=4, label_target_size=388) steps_per_epoch = next(training_gen) validation_steps = next(validation_gen) model = UNet(input_size=(572, 572, 3)) metrics = [ Accuracy(), CategoricalAccuracy(), CategoricalCrossentropy(), ArgmaxMeanIoU(num_classes=6, name="mean_iou") ] optimizer = SGD(learning_rate=initial_learning_rate, momentum=0.99, nesterov=True) model.compile(optimizer=optimizer, loss=categorical_crossentropy, metrics=metrics) start_time = int(time.time()) os.mkdir(f"weights/{start_time}_{model.name}/") metrics_to_log = [ "loss", "accuracy", "categorical_accuracy", "mean_iou", "categorical_crossentropy" ] model_path = f"weights/{start_time}_{model.name}/" callbacks = [ save_model_on_epoch_end(model.name, model, model_path), metrics_to_csv_logger(model_path + "/batch.csv", metrics_to_log), CSVLogger(model_path + "/epoch.csv", separator=";"), LearningRateScheduler(lr_schedule(initial_lr=initial_learning_rate)), ] model.fit(training_gen, epochs=20, steps_per_epoch=steps_per_epoch, validation_data=validation_gen, validation_steps=validation_steps, callbacks=callbacks)
def build_classifier(pretrained_model, num_labels, optimizer, options): seq_len = options.seq_len input_ids = Input( shape=(seq_len,), dtype='int32', name='input_ids') # token_type_ids = Input( # shape=(seq_len,), dtype='int32', name='token_type_ids') attention_mask = Input( shape=(seq_len,), dtype='int32', name='attention_mask') # inputs = [input_ids, attention_mask, token_type_ids] inputs = [input_ids, attention_mask] pretrained_outputs = pretrained_model(inputs) #pooled_output = pretrained_outputs[1] pooled_output = pretrained_outputs['last_hidden_state'][:,0,:] #CLS # TODO consider Dropout here if options.multiclass: output = Dense(num_labels, activation='softmax')(pooled_output) loss = CategoricalCrossentropy() metrics = [CategoricalAccuracy(name='acc')] else: output = Dense(num_labels, activation='sigmoid')(pooled_output) loss = BinaryCrossentropy() metrics = [ #F1Score(name='f1_th0.3', num_classes=num_labels, average='micro', threshold=0.3), F1Score(name='f1_th0.4', num_classes=num_labels, average='micro', threshold=0.4)#, #F1Score(name='f1_th0.5', num_classes=num_labels, average='micro', threshold=0.5), #AUC(name='auc', multi_label=True) ] #output = pretrained_outputs # test model = Model( inputs=inputs, outputs=[output] ) model.compile( optimizer=optimizer, loss=loss, metrics=metrics ) return model
def build_model(max_seq_length): in_id = tf.keras.layers.Input(shape=(max_seq_length,), name="input_ids") in_mask = tf.keras.layers.Input(shape=(max_seq_length,), name="input_masks") in_segment = tf.keras.layers.Input(shape=(max_seq_length,), name="segment_ids") bert_inputs = [in_id, in_mask, in_segment] bert_output = BertLayer(n_fine_tune_layers=10, pooling="first")(bert_inputs) dense = tf.keras.layers.Dense(512, activation='relu')(bert_output) pred = tf.keras.layers.Dense(12, activation='softmax')(dense) model = tf.keras.models.Model(inputs=bert_inputs, outputs=pred) optimizer = Adam(learning_rate=5e-05,epsilon=1e-08,decay=0.01,clipnorm=1.0) loss = CategoricalCrossentropy(from_logits = True) metric = CategoricalAccuracy('accuracy') model.compile(loss=loss, optimizer=optimizer, metrics=[metric])#['accuracy']) model.summary() return model
def define_model(self): inputs = tf.keras.Input(shape=(n_inputs, 1), name='input') # 64 filters, 10 kernel size x = Conv1D(64, 10, activation='relu')(inputs) x = MaxPool1D()(x) x = BatchNormalization()(x) x = Conv1D(128, 10, activation='relu')(x) x = MaxPool1D()(x) x = BatchNormalization()(x) x = Conv1D(128, 10, activation='relu')(x) x = MaxPool1D()(x) x = BatchNormalization()(x) x = Conv1D(256, 10, activation='relu')(x) x = MaxPool1D()(x) x = BatchNormalization()(x) x = Flatten()(x) x = Dense(1024, activation='relu', name='dense_1')(x) x = BatchNormalization()(x) x = Dropout(dropout)(x) x = Dense(2048, activation='relu', name='dense_2')(x) x = BatchNormalization()(x) x = Dropout(dropout)(x) outputs = Dense(n_classes, activation='softmax', name='predictions')(x) self.cnn_model = tf.keras.Model(inputs=inputs, outputs=outputs) optimizer = tf.keras.optimizers.Adam(lr=learning_rate) accuracy = CategoricalAccuracy() self.cnn_model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=[accuracy])
def do_training(initial_learning_rate=0.001): gids = get_gids_from_database("wnet") training_gen, validation_gen = initialize_train_and_validation_generators( "wnet", gids, batch_size=5, label_target_size=256, use_image_as_label=True) steps_per_epoch = next(training_gen) validation_steps = next(validation_gen) full_model, encoder_model = WNet(nb_classes=6) metrics = [Accuracy(), CategoricalAccuracy(), CategoricalCrossentropy()] optimizer = Adam(lr=initial_learning_rate) full_model.compile(optimizer=optimizer, loss=categorical_crossentropy, metrics=metrics) start_time = int(time.time()) model_path = f"weights/{start_time}_{full_model.name}/" os.mkdir(model_path) metrics_to_log = [metric.name for metric in metrics] callbacks = [ save_model_on_epoch_end(full_model.name, full_model, model_path), save_model_on_epoch_end(encoder_model.name, encoder_model, model_path), metrics_to_csv_logger(model_path + "/batch.csv", ["loss"] + metrics_to_log), CSVLogger(model_path + "/epoch.csv", separator=";"), ] full_model.fit(training_gen, epochs=1, steps_per_epoch=steps_per_epoch, validation_data=validation_gen, validation_steps=validation_steps, callbacks=callbacks)
from tensorflow.keras.losses import CategoricalCrossentropy from tensorflow.keras.metrics import Mean, CategoricalAccuracy from user_profile_prediction.etl.embedding import Embedding from user_profile_prediction.etl.preprocess_train_data import PreprocessTrainingData from user_profile_prediction.training.deep_learning_model_train_step import DeepLearningModelTraining p: PreprocessTrainingData = PreprocessTrainingData( "/Volumes/Samsung_T5/Files/Document/china_hadoop/GroupProject/project_data/data/train.csv", embedding_size=500, sentence_len=400) p.split_sentence() e = Embedding(500, 10) m = e.train_word2vec_model(p.sentences_with_split_words) # m = e.load_embedding_model() x_train, x_val, y_train, y_val = p.split_data(p.age_data_iter(e)) text_rnn = TextRNN(400, 500, 6) optimizer: Adam = Adam(learning_rate=1e-4) losses: CategoricalCrossentropy = CategoricalCrossentropy() metric = CategoricalAccuracy() step = DeepLearningModelTraining(text_rnn, e, optimizer, losses, metric) step.build((None, 400, 500)) step.compile() step.fit(x_train, y_train, x_val, y_val, 500, 50)
def train(self, x_train_main, x_train_aux, y_train, x_val, y_val, auxdt_size, num_layers_intnet, batch_size, epochs_prior, epochs_perc, epochs_adj, decay, history): # 1:事前学習 x_train = np.concatenate([x_train_main, x_train_aux], axis=1) self.model_wholenet.summary() self.model_wholenet.fit(x_train, y_train, epochs=epochs_prior, batch_size=batch_size, verbose=self.verbose, validation_data=(x_val, y_val), callbacks=[history]) # 2:浸透学習 epoch = 0 loss = 1 non_perc_rate = 1 # 非浸透率の初期値 nprate_min = 1e-8 # 非浸透率の閾値 loss_min = 1e-5 # 損失関数の値の閾値 # 統合サブネットの重みを固定 for i in range(3 * (num_layers_intnet - 1) + 2): self.model_wholenet.layers[-i-1].trainable = False self.model_wholenet.compile(optimizer=self.optimizer, loss=categorical_crossentropy, metrics=[CategoricalAccuracy(), Precision(), Recall()]) # 浸透学習 self.model_wholenet.summary() while epoch < epochs_perc: non_perc_rate = (1 - decay) ** epoch x_train[:, -auxdt_size:] *= (1 - decay) print('Non-Percolation Rate =', non_perc_rate) self.model_wholenet.fit(x_train, y_train, initial_epoch=epochs_prior + epoch, epochs=epochs_prior + epoch + 1, batch_size=batch_size, verbose=self.verbose, validation_data=(x_val, y_val), callbacks=[history]) epoch += 1 # 3:微調整 # 統合サブネットの重み固定を解除 for i in range(3 * (num_layers_intnet - 1) + 2): self.model_wholenet.layers[-i-1].trainable = True self.model_wholenet.compile(optimizer=self.optimizer, loss=categorical_crossentropy, metrics=[CategoricalAccuracy(), Precision(), Recall()]) self.model_wholenet.summary() if True: # 微調整を行う条件を入れる(現状は常に微調整を行う) non_perc_rate = 0 x_train_aux *= non_perc_rate x_train = np.concatenate([x_train_main, x_train_aux], axis=1) self.model_wholenet.fit(x_train, y_train, initial_epoch=epochs_prior + epochs_perc, epochs=epochs_prior + epoch + epochs_adj, batch_size=batch_size, verbose=self.verbose, validation_data=(x_val, y_val), callbacks=[history]) return history
def objective(trial: optuna.Trial): base_lr = trial.suggest_float("base_lr", 0.001, 0.05, step=0.001) weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1e-3) ema = trial.suggest_categorical("ema", ["true", "false"]) ema_decay = trial.suggest_loguniform("ema_decay", 0.99, 0.9999) if ema == 'true' else None @curry def transform(image, label, training): image = pad(image, 2) image, label = to_tensor(image, label) image = normalize(image, [0.1307], [0.3081]) label = tf.one_hot(label, 10) return image, label batch_size = 128 eval_batch_size = 256 ds_train, ds_test, steps_per_epoch, test_steps = make_mnist_dataset( batch_size, eval_batch_size, transform, sub_ratio=0.01) model = LeNet5() model.build((None, 32, 32, 1)) criterion = CrossEntropy() epochs = 20 lr_shcedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_shcedule, momentum=0.9, nesterov=True, weight_decay=weight_decay) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir=f"./MNIST", multiple_steps=True) callbacks = [OptunaReportIntermediateResult('acc', trial)] # if ema == 'true': # callbacks.append(EMA(ema_decay)) learner.fit(ds_train, epochs, ds_test, val_freq=2, steps_per_epoch=steps_per_epoch, val_steps=test_steps, callbacks=callbacks) return learner.metric_history.get_metric('acc', "eval")[-1]
class ExperimentClassify: def __init__(self, model, optimizer, exptConfig): self.now = dt.now().strftime('%Y-%m-%d--%H-%M-%S') self.exptConfig = exptConfig self.model = model self.optimizer = optimizer self.loss = CategoricalCrossentropy( from_logits=exptConfig['LossParams']['fromLogits']) # ------------ metrics ---------------------- self.catAccTest = CategoricalAccuracy() self.catAccTrain = CategoricalAccuracy() self.exptFolder = os.path.join( exptConfig['OtherParams']['exptBaseFolder'], self.now, exptConfig['ModelParams']['name']) self.modelFolder = os.path.join(self.exptFolder, 'model') self.chkptFolder = os.path.join(self.exptFolder, 'checkpoints') os.makedirs(self.modelFolder, exist_ok=True) os.makedirs(self.chkptFolder, exist_ok=True) self.stepNumber = 0 self.evalNumber = 0 self.epoch = 0 # All the logs go here ... # ------------------------ self.createMetaData() self.logDir = os.path.join(self.exptFolder, 'logs') self.scalarWriter = tf.summary.create_file_writer( os.path.join(self.logDir, 'scalars', 'metrics')) self.graphWriter = tf.summary.create_file_writer( os.path.join(self.logDir, 'graph')) return def step(self, x, y): with tf.GradientTape() as tape: yHat = self.model.call(x) loss = self.loss(y, yHat) grads = tape.gradient(loss, self.model.trainable_weights) self.optimizer.apply_gradients( zip(grads, self.model.trainable_weights)) self.catAccTrain.update_state(y, yHat) with self.scalarWriter.as_default(): tf.summary.scalar('training loss', data=loss, step=self.stepNumber) tf.summary.scalar('training accuracy', data=self.catAccTrain.result().numpy(), step=self.stepNumber) self.stepNumber += 1 return loss.numpy() def eval(self, x, y): yHat = self.model.predict(x) self.catAccTest.update_state(y, yHat) with self.scalarWriter.as_default(): tf.summary.scalar('testing accuracy', data=self.catAccTest.result().numpy(), step=self.evalNumber) self.evalNumber += 1 return self.catAccTest.result().numpy() def createMetaData(self): if not os.path.exists(self.exptFolder): os.makedirs(self.exptFolder) with open(os.path.join(self.exptFolder, 'config.json'), 'w') as fOut: json.dump(self.exptConfig, fOut) return def createModelSummary(self, x): tf.summary.trace_on(graph=True) self.model.predict(x) with self.graphWriter.as_default(): tf.summary.trace_export('name', step=0) tf.summary.trace_off() def saveModel(self): try: self.model.save(self.modelFolder) except Exception as e: print(f'Unable to save the model: {e}') return def checkPoint(self): try: epoch = self.epoch step = self.stepNumber self.model.save_weights( os.path.join(self.chkptFolder, f'{epoch:07d}-{step:07d}')) except Exception as e: print(f'Unable to checkpoint: {self.stepNumber}: {e}') return
def training_model(training_file_path: str, embedding_size: int, sentence_len: int, conv_filter: int, global_max_pool: bool, pool_size: int, drop_rate: float, dense_size: int, l1_regularization: float, l2_regularization: float, vocabulary_size: int, min_count: int, class_num: int, label_name: int, learning_rate: int, epochs: int, batch_size: int, checkpoint_saved_path: str): global TRAINING_FILE_PATH, EMBEDDING_SIZE, SENTENCE_LEN, MIN_COUNT, CLASS_NUM, LEARNING_RATE,\ EPOCHS, BATCH_SIZE, MODEL, VOCABULARY_SIZE TRAINING_FILE_PATH = training_file_path EMBEDDING_SIZE = embedding_size SENTENCE_LEN = sentence_len VOCABULARY_SIZE = vocabulary_size MIN_COUNT = min_count CLASS_NUM = class_num LEARNING_RATE = learning_rate EPOCHS = epochs BATCH_SIZE = batch_size preprocess: PreprocessTrainingData = PreprocessTrainingData( csv_file_path=TRAINING_FILE_PATH, embedding_size=EMBEDDING_SIZE, sentence_len=SENTENCE_LEN, vocabulary_size=VOCABULARY_SIZE) preprocess.split_sentence() tokenizer = preprocess.tokenize() x_train, x_val, y_train, y_val = preprocess.split_data( preprocess.age_data_iter()) text_cnn = TextCNN(SENTENCE_LEN, EMBEDDING_SIZE, CLASS_NUM, VOCABULARY_SIZE, conv_filter, global_max_pool, pool_size, drop_rate, dense_size, l1_regularization, l2_regularization) optimizer: Adam = Adam(learning_rate=LEARNING_RATE) losses: CategoricalCrossentropy = CategoricalCrossentropy() metric = CategoricalAccuracy() training = DeepLearningModelTraining(text_cnn, optimizer, losses, metric) training.build(( None, SENTENCE_LEN, )) training.compile() log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1) training.fit(x_train, y_train, x_val, y_val, epochs=EPOCHS, batch=BATCH_SIZE, callbacks=[tensorboard_callback]) ckpt = tf.train.Checkpoint(text_cnn=training.training_model) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_saved_path, max_to_keep=5) path = ckpt_manager.save() print("checkpoint of text_cnn has been saved in {}".format(path))
inputs = Input(shape=input_shape) teacher = KDModel.Teacher(NUM_CLASSES) teacher_model = teacher.createModel(inputs) # Teacherモデルの学習 optimizer = Adamax(learning_rate=LR_T) # 最適化アルゴリズム history_teacher = LossAccHistory() training = KDModel.NormalTraining(teacher_model) teacher_model.summary() # plot_model(teacher_model, show_shapes=True, to_file='teacher_model.png') for epoch in range(1, EPOCHS_T + 1): loss_metric = Mean() loss_metric_val = Mean() acc_metric = Mean() acc_metric_val = Mean() acc = CategoricalAccuracy() acc_val = CategoricalAccuracy() # 各バッチごとに学習 for (x_train_, y_train_), (x_val_, y_val_) in ds: loss_value, grads = training.grad(x_train_, y_train_) optimizer.apply_gradients(zip(grads, teacher_model.trainable_weights)) loss_value_test = training.loss(x_val_, y_val_) probs = tf.nn.softmax(teacher_model(x_train_)) probs_val = tf.nn.softmax(teacher_model(x_val_)) loss_metric(loss_value) acc_metric(acc(y_train_, probs)) loss_metric_val(loss_value_test) acc_metric_val(acc_val(y_val_, probs_val))
### ------- Train the model ------- ### # Set an optimizer optimizer = Adam(learning_rate=5e-05, epsilon=1e-08, decay=0.01, clipnorm=1.0) # Set loss and metrics loss = { 'gia': CategoricalCrossentropy(from_logits=True), 'dich_vu': CategoricalCrossentropy(from_logits=True), 'an_toan': CategoricalCrossentropy(from_logits=True), 'chat_luong': CategoricalCrossentropy(from_logits=True), 'ship': CategoricalCrossentropy(from_logits=True), 'chinh_hang': CategoricalCrossentropy(from_logits=True) } metric = { 'gia': CategoricalAccuracy('accuracy'), 'dich_vu': CategoricalAccuracy('accuracy'), 'an_toan': CategoricalAccuracy('accuracy'), 'chat_luong': CategoricalAccuracy('accuracy'), 'ship': CategoricalAccuracy('accuracy'), 'chinh_hang': CategoricalAccuracy('accuracy') } # Compile the model model.compile(optimizer=optimizer, loss=loss, metrics=metric) # Ready output data for the model y_gia = to_categorical(data['giá']) y_dich_vu = to_categorical(data['dịch_vụ']) y_an_toan = to_categorical(data['an_toàn']) y_chat_luong = to_categorical(data['chất_lượng'])
# train, val, eval 데이터셋으로 변환 train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(10000).batch(32) val_ds = tf.data.Dataset.from_tensor_slices((X_val, y_val)).batch(32) # model 생성 model = MyModel(config) # loss 함수와 옵티마이저 설정 loss_object = CategoricalCrossentropy() # y label이 one-hot encoding optimizer = Adam() # train, val의 loss와 accuracy 설정 train_loss = Mean(name='train_loss') train_accuracy = CategoricalAccuracy(name='train_accuracy') val_loss = Mean(name='val_loss') val_accuracy = CategoricalAccuracy(name='val_accuracy') ######################################################################## ## train ######################################################################## print('\n\n*** 학습을 시작합니다...') ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model) manager = tf.train.CheckpointManager(checkpoint=ckpt, directory='./tf_ckpts', max_to_keep=config['epochs'])
# Then build your model output author = Dense( units=len(data.author.value_counts()), kernel_initializer=TruncatedNormal(stddev=config.initializer_range), name='author')(pooled_output) outputs = {'author': author} # And combine it all in a model object model = Model(inputs=inputs, outputs=outputs, name='BERT_MultiClass') # Take a look at the model model.summary() # Set an optimizer optimizer = Adam(learning_rate=5e-05, epsilon=1e-08, decay=0.01, clipnorm=1.0) # Set loss and metrics loss = {'author': CategoricalCrossentropy(from_logits=True)} metric = {'author': CategoricalAccuracy('accuracy')} # Compile the model model.compile(optimizer=optimizer, loss=loss, metrics=metric) # Ready output data for the model y_author = to_categorical(data['author']) # Tokenize the input (takes some time) x = tokenizer(text=data['text_without_stopwords'].to_list(), add_special_tokens=True, max_length=max_length, truncation=True, padding=True, return_tensors='tf', return_token_type_ids=False, return_attention_mask=False, verbose=True)
model.summary() # -----------------------------------------------------------------# # Train the model # Set an optimizer optimizer = Adam( # learning_rate=5e-05, learning_rate=1e-5, epsilon=1e-08, decay=0.01, clipnorm=1.0) # Set loss and metrics loss = {'classification': CategoricalCrossentropy(from_logits=True)} metric = {'classification': CategoricalAccuracy('accuracy')} # Compile the model model.compile(optimizer=optimizer, loss=loss, metrics=metric) # Ready output data for the model y_classification = to_categorical(data['Classification']) # Tokenize the input (takes some time) x = tokenizer(text=data['Comments'].to_list(), add_special_tokens=True, max_length=max_length, truncation=True, padding=True, return_tensors='tf', return_token_type_ids=False,
training_iterator = data_generator.flow(x_train, y_train, batch_size=Batch_size) validation_iterator = data_generator.flow(x_valid, y_valid, batch_size=Batch_size) # Build the model model = Sequential() model.add(Input(shape=(128, 128, 3))) #refer to input_data.shape # Adding two convolutional layers, interspersed with max pooling layers, followed by two dense layers: model.add(Conv2D(8, 3, strides=2, activation='relu')) model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2))) model.add(Conv2D(8, 3, strides=2, activation='relu')) model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2))) model.add(Flatten()) model.add(Dense(16, activation='relu')) # Output model.add(Dense(4, activation='softmax')) model.compile(optimizer=Adam(learning_rate=0.001), loss=CategoricalCrossentropy(), metrics=[CategoricalAccuracy(),AUC()]) # Because the labels are one-hot categories, use tf.keras.losses.CategoricalCrossentropy() as your loss. print(model.summary()) # Train model.fit(training_iterator, steps_per_epoch=x_train.shape[0]/Batch_size, epochs=8, validation_data=validation_iterator, validation_steps=y_train.shape[0]/Batch_size) #can be len(x_train) instead # model’s accuracy should be around 0.60-0.70, and your AUC should fall into the 0.80-0.90 range: val_categorical_accuracy: 0.6152 - val_auc: 0.8441 # Your accuracy tells you that your model assigns the highest probability to the correct class more than 60% of the time. For a classification task with over four classes, this is no small feat: a random baseline model would achieve only ~25% accuracy on the dataset. Your AUC tells you that for a random galaxy, there is more than an 80% chance your model would assign a higher probability to a true class than to a false one. #If you would like, try tweaking your architecture. Can you find a better set of hyperparameters? MAKE SURE to watch your parameter count: it’s easy to accidentally create a model with more than tens of thousands of parameters, which could overfit to your relatively small dataset (or crash the Learning Environment). Note that scores will fluctuate a bit, depending on how the weights are randomly initialized. # learning rate # number of convolutional layers # number of filters, strides, and padding type per layer # stride and pool_size of max pooling layers # size of hidden linear layers
def train_network(model, train_x, train_y, val_x, val_y, batch_s, epochs1, epochs2): """ Function to train network in two steps: * Train network with initial VGG base layers frozen * Unfreeze all layers and retrain with smaller learning rate :param model: CNN model :param train_x: training input :param train_y: training outputs :param val_x: validation inputs :param val_y: validation outputs :param batch_s: batch size :param epochs1: epoch count for initial training :param epochs2: epoch count for training all layers unfrozen :return: trained network """ # Freeze VGG19 pre-trained layers. if config.imagesize == "large": model.layers[0].layers[1].trainable = False else: model.layers[0].trainable = False # Train model with frozen layers (all training with early stopping dictated by loss in validation over 3 runs). if config.dataset == "mini-MIAS": model.compile(optimizer=Adam(1e-3), loss=CategoricalCrossentropy(), metrics=[CategoricalAccuracy()]) hist_1 = model.fit(x=train_x, y=train_y, batch_size=batch_s, steps_per_epoch=len(train_x) // batch_s, validation_data=(val_x, val_y), validation_steps=len(val_x) // batch_s, epochs=epochs1, callbacks=[ EarlyStopping( monitor='val_categorical_accuracy', patience=8, restore_best_weights=True), ReduceLROnPlateau(patience=4) ]) elif config.dataset == "CBIS-DDSM": model.compile(optimizer=Adam(lr=1e-4), loss=BinaryCrossentropy(), metrics=[BinaryAccuracy()]) hist_1 = model.fit(x=train_x, validation_data=val_x, epochs=epochs1, callbacks=[ EarlyStopping(monitor='val_loss', patience=8, restore_best_weights=True), ReduceLROnPlateau(patience=6) ]) # Plot the training loss and accuracy. plot_training_results(hist_1, "Initial_training", True) # Train a second time with a smaller learning rate and with all layers unfrozen # (train over fewer epochs to prevent over-fitting). if config.imagesize == "large": model.layers[0].layers[1].trainable = True else: model.layers[0].trainable = True if config.dataset == "mini-MIAS": model.compile( optimizer=Adam(1e-5), # Very low learning rate loss=CategoricalCrossentropy(), metrics=[CategoricalAccuracy()]) hist_2 = model.fit(x=train_x, y=train_y, batch_size=batch_s, steps_per_epoch=len(train_x) // batch_s, validation_data=(val_x, val_y), validation_steps=len(val_x) // batch_s, epochs=epochs2, callbacks=[ EarlyStopping( monitor='val_categorical_accuracy', patience=8, restore_best_weights=True), ReduceLROnPlateau(patience=6) ]) elif config.dataset == "CBIS-DDSM": model.compile( optimizer=Adam(lr=1e-5), # Very low learning rate loss=BinaryCrossentropy(), metrics=[BinaryAccuracy()]) hist_2 = model.fit(x=train_x, validation_data=val_x, epochs=epochs2, callbacks=[ EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True), ReduceLROnPlateau(patience=6) ]) # Plot the training loss and accuracy. plot_training_results(hist_2, "Fine_tuning_training", False) return model
def generate_compiled_segmentation_model( model_name, model_parameters, num_classes, loss, optimizer, weights_to_load=None, optimizing_threshold_class_metric=None, optimizing_class_id=None, optimizing_input_threshold=None, optimized_class_thresholds=None): # These are the only model, loss, and optimizer currently supported assert model_name == 'Unet' assert loss == 'cross_entropy' assert optimizer == 'adam' loss_fn = BinaryCrossentropyL() all_metrics = [ ] # one-hot versions are generally preferred for given metric # make first metric a copy of loss, to continually verify `val_loss` is correct if isinstance(loss_fn, BinaryCrossentropyL): all_metrics.append(BinaryCrossentropyM(name='binary_ce_metric')) else: all_metrics.append(CategoricalCrossentropyM(name='categ_ce_metric')) # standard thresholded version (default threshold is 0.5) also kept below, in case it's desired in certain scenario for class_num in range(num_classes + 1): if class_num == 0 and optimizing_threshold_class_metric is None: # all class metrics # note, `loss_fn` for all classes placed before `all_metrics` in lineup of command window metrics and plots if not isinstance(loss_fn, BinaryCrossentropyL): all_metrics.extend([CategoricalCELoss()]) all_metrics[1].name = str('categ_cross_entropy_sm') all_metrics.extend([ AccuracyTfKeras(), # OneHotAccuracyTfKeras(), # `global_threshold` built-in ClassBinaryAccuracyTfKeras(thresholds=global_threshold), # OneHotClassBinaryAccuracyTfKeras(thresholds=global_threshold), ClassBinaryAccuracySM(threshold=global_threshold), # OneHotClassBinaryAccuracySM(threshold=global_threshold), BinaryAccuracy(threshold=global_threshold), CategoricalAccuracy(), FalseNegatives(name='false_neg', thresholds=global_threshold), # OneHotFalseNegatives(name='false_neg_1H', thresholds=global_threshold), TrueNegatives(name='true_neg', thresholds=global_threshold), # OneHotTrueNegatives(name='true_neg_1H', thresholds=global_threshold), FalsePositives(name='false_pos', thresholds=global_threshold), # OneHotFalsePositives(name='false_pos_1H', thresholds=global_threshold), TruePositives(name='true_pos', thresholds=global_threshold), # OneHotTruePositives(name='true_pos_1H', thresholds=global_threshold), Recall(name='recall', thresholds=global_threshold), # OneHotRecall(name='recall_1H', thresholds=global_threshold), Precision(name='precision', thresholds=global_threshold), # OneHotPrecision(name='precision_1H', thresholds=global_threshold), FBetaScore(name='f1_score', beta=1, thresholds=global_threshold), # OneHotFBetaScore(name='f1_score_1H', beta=1, thresholds=global_threshold), IoUScore(name='iou_score', thresholds=global_threshold), # OneHotIoUScore(name='iou_score_1H', thresholds=global_threshold) ]) elif class_num == 0 and optimizing_threshold_class_metric is not None: # all class metrics continue else: # per class metrics if optimizing_threshold_class_metric is not None: class_threshold = optimizing_input_threshold class_num = optimizing_class_id + 1 elif optimized_class_thresholds is None: class_threshold = global_threshold else: class_threshold = optimized_class_thresholds[str( 'class' + str(class_num - 1))] all_metrics.append(CategoricalCELoss(class_indexes=class_num - 1)) all_metrics[-1].name = str('class' + str(class_num - 1) + '_binary_cross_entropy') all_metrics.append( ClassBinaryAccuracySM(name=str('class' + str(class_num - 1) + '_binary_accuracy_sm'), class_indexes=class_num - 1, threshold=class_threshold)) all_metrics.append( ClassBinaryAccuracyTfKeras( name=str('class' + str(class_num - 1) + '_binary_accuracy_tfkeras'), class_id=class_num - 1, thresholds=class_threshold)) all_metrics.append( IoUScore(name=str('class' + str(class_num - 1) + '_iou_score'), class_id=class_num - 1, thresholds=class_threshold)) all_metrics.append( FBetaScore(name=str('class' + str(class_num - 1) + '_f1_score'), class_id=class_num - 1, beta=1, thresholds=class_threshold)) all_metrics.append( Precision(name=str('class' + str(class_num - 1) + '_precision'), class_id=class_num - 1, thresholds=class_threshold)) all_metrics.append( Recall(name=str('class' + str(class_num - 1) + '_recall'), class_id=class_num - 1, thresholds=class_threshold)) if optimizing_threshold_class_metric is not None: break if num_classes == 1: break # strategy = tf.distribute.MirroredStrategy() # with strategy.scope(): model = Unet(input_shape=(None, None, 1), classes=num_classes, **model_parameters) model.compile(optimizer=Adam(), loss=loss_fn, metrics=all_metrics) if weights_to_load: model.load_weights(weights_to_load) if optimizing_threshold_class_metric is None: print(model.summary()) return model
'/content/test1', class_mode='categorical', color_mode='grayscale', batch_size=BATCH_SIZE) model = tf.keras.Sequential() model.add(tf.keras.Input(shape=(256, 256, 1))) model.add(tf.keras.layers.Conv2D(8, 2, strides=2, activation='relu')) model.add(tf.keras.layers.MaxPooling2D(pool_size=(5, 5), strides=(5, 5))) model.add(tf.keras.layers.Conv2D(8, 2, strides=2, activation='relu')) model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(64, activation='relu')) model.add(tf.keras.layers.Dense(3, activation='softmax')) model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss=tf.keras.losses.CategoricalCrossentropy(), metrics=[CategoricalAccuracy(), AUC()]) model.summary() stop = EarlyStopping(monitor='categorical_accuracy', mode='max', verbose=1, patience=3) model.fit(training_iterator, steps_per_epoch=training_iterator.samples / BATCH_SIZE, epochs=15, validation_data=validation_iterator, validation_steps=validation_iterator.samples / BATCH_SIZE, callbacks=[stop]) to_predict = ImageDataGenerator().flow_from_directory('/content/predictcovid',
A_in = Input(shape=(None,), sparse=True) I_in = Input(shape=(), name='segment_ids_in', dtype=tf.int32) X_1 = GraphConvSkip(32, activation='relu')([X_in, A_in]) X_1, A_1, I_1 = TopKPool(ratio=0.5)([X_1, A_in, I_in]) X_2 = GraphConvSkip(32, activation='relu')([X_1, A_1]) X_2, A_2, I_2 = TopKPool(ratio=0.5)([X_2, A_1, I_1]) X_3 = GraphConvSkip(32, activation='relu')([X_2, A_2]) X_3 = GlobalAvgPool()([X_3, I_2]) output = Dense(n_out, activation='softmax')(X_3) # Build model model = Model(inputs=[X_in, A_in, I_in], outputs=output) opt = Adam(lr=learning_rate) loss_fn = CategoricalCrossentropy() acc_fn = CategoricalAccuracy() @tf.function( input_signature=(tf.TensorSpec((None, F), dtype=tf.float64), tf.SparseTensorSpec((None, None), dtype=tf.float32), tf.TensorSpec((None,), dtype=tf.int32), tf.TensorSpec((None, n_out), dtype=tf.float64)), experimental_relax_shapes=True) def train_step(X_, A_, I_, y_): with tf.GradientTape() as tape: predictions = model([X_, A_, I_], training=True) loss = loss_fn(y_, predictions) loss += sum(model.losses) acc = acc_fn(y_, predictions) gradients = tape.gradient(loss, model.trainable_variables)
ds = tf.data.Dataset.zip((ds_train, ds_val)) # Teacherモデルの定義 inputs_main = Input(shape=input_shape_main) inputs_aux = Input(shape=input_shape_aux) teacher = KDModel.Teacher(NUM_CLASSES, T) teacher_model = teacher.createModel(inputs_main, inputs_aux) # Teacherモデルの学習 training = KDModel.NormalTraining(teacher_model) teacher_model.summary() # plot_model(teacher_model, show_shapes=True, to_file='teacher_model.png') for epoch in range(1, EPOCHS_T + 1): epoch_loss_avg = Mean() epoch_loss_avg_val = Mean() epoch_accuracy = CategoricalAccuracy() epoch_accuracy_val = CategoricalAccuracy() # 各バッチごとに学習 for (x_train_main_, x_train_aux_, y_train_), (x_val_main_, x_val_aux_, y_val_) in ds: loss_value, grads = training.grad([x_train_main_, x_train_aux_], y_train_) optimizer.apply_gradients(zip(grads, teacher_model.trainable_variables)) loss_value_test = training.loss([x_val_main_, x_val_aux_], y_val_) epoch_loss_avg(loss_value) epoch_accuracy( y_train_, tf.nn.softmax(teacher_model([x_train_main_, x_train_aux_])))
drop_path = 0.3 model = NASNet(16, 8, True, drop_path, 10, PC_DARTS_cifar) model.build((None, 32, 32, 3)) model.summary() criterion = CrossEntropy(auxiliary_weight=0.4) base_lr = 0.025 epochs = 600 lr_schedule = CosineLR(base_lr * mul, steps_per_epoch, epochs=epochs, min_lr=0) optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=3e-4, nesterov=True) train_metrics = { 'loss': Mean(), 'acc': CategoricalAccuracy(), } eval_metrics = { 'loss': CategoricalCrossentropy(from_logits=True), 'acc': CategoricalAccuracy(), } learner = SuperLearner(model, criterion, optimizer, grad_clip_norm=5.0, jit_compile=False, train_metrics=train_metrics, eval_metrics=eval_metrics, work_dir="./CIFAR10-PC-DARTS")
def train_model(cfg, data, callbacks, verbose=1): ''' Train a and evaluate model on given data. :param cfg: Project config (from config.yml) :param data: dict of partitioned dataset :param callbacks: list of callbacks for Keras model :param verbose: Verbosity mode to pass to model.fit_generator() :return: Trained model and associated performance metrics on the test set ''' # If set in config file, oversample the minority class if cfg['TRAIN']['IMB_STRATEGY'] == 'random_oversample': data['TRAIN'] = random_minority_oversample(data['TRAIN']) # Create ImageDataGenerators train_img_gen = ImageDataGenerator(rotation_range=10, preprocessing_function=remove_text, samplewise_std_normalization=True, samplewise_center=True) val_img_gen = ImageDataGenerator(preprocessing_function=remove_text, samplewise_std_normalization=True, samplewise_center=True) test_img_gen = ImageDataGenerator(preprocessing_function=remove_text, samplewise_std_normalization=True, samplewise_center=True) # Create DataFrameIterators img_shape = tuple(cfg['DATA']['IMG_DIM']) y_col = 'label_str' class_mode = 'categorical' train_generator = train_img_gen.flow_from_dataframe(dataframe=data['TRAIN'], directory=cfg['PATHS']['RAW_DATA'], x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'], class_mode=class_mode, validate_filenames=False) val_generator = val_img_gen.flow_from_dataframe(dataframe=data['VAL'], directory=cfg['PATHS']['RAW_DATA'], x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'], class_mode=class_mode, validate_filenames=False) test_generator = test_img_gen.flow_from_dataframe(dataframe=data['TEST'], directory=cfg['PATHS']['RAW_DATA'], x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'], class_mode=class_mode, validate_filenames=False, shuffle=False) # Save model's ordering of class indices dill.dump(test_generator.class_indices, open(cfg['PATHS']['OUTPUT_CLASS_INDICES'], 'wb')) # Apply class imbalance strategy. We have many more X-rays negative for COVID-19 than positive. histogram = np.bincount(np.array(train_generator.labels).astype(int)) # Get class distribution class_weight = None if cfg['TRAIN']['IMB_STRATEGY'] == 'class_weight': class_multiplier = cfg['TRAIN']['CLASS_MULTIPLIER'] class_multiplier = [class_multiplier[cfg['DATA']['CLASSES'].index(c)] for c in test_generator.class_indices] class_weight = get_class_weights(histogram, class_multiplier) # Define metrics. covid_class_idx = test_generator.class_indices['COVID-19'] # Get index of COVID-19 class thresholds = 1.0 / len(cfg['DATA']['CLASSES']) # Binary classification threshold for a class metrics = ['accuracy', CategoricalAccuracy(name='accuracy'), Precision(name='precision', thresholds=thresholds, class_id=covid_class_idx), Recall(name='recall', thresholds=thresholds, class_id=covid_class_idx), AUC(name='auc'), F1Score(name='f1score', thresholds=thresholds, class_id=covid_class_idx)] # Define the model. print('Training distribution: ', ['Class ' + list(test_generator.class_indices.keys())[i] + ': ' + str(histogram[i]) + '. ' for i in range(len(histogram))]) input_shape = cfg['DATA']['IMG_DIM'] + [3] num_gpus = cfg['TRAIN']['NUM_GPUS'] if cfg['TRAIN']['MODEL_DEF'] == 'dcnn_resnet': model_def = dcnn_resnet elif cfg['TRAIN']['MODEL_DEF'] == 'resnet50v2': model_def = resnet50v2 else: model_def = resnet101v2 if cfg['TRAIN']['CLASS_MODE'] == 'binary': histogram = np.bincount(data['TRAIN']['label'].astype(int)) output_bias = np.log([histogram[i] / (np.sum(histogram) - histogram[i]) for i in range(histogram.shape[0])]) model = model_def(cfg['NN']['DCNN_BINARY'], input_shape, metrics, 2, output_bias=output_bias, gpus=num_gpus) else: n_classes = len(cfg['DATA']['CLASSES']) histogram = np.bincount(data['TRAIN']['label'].astype(int)) output_bias = np.log([histogram[i] / (np.sum(histogram) - histogram[i]) for i in range(histogram.shape[0])]) model = model_def(cfg['NN']['DCNN_MULTICLASS'], input_shape, metrics, n_classes, output_bias=output_bias, gpus=num_gpus) # Train the model. steps_per_epoch = ceil(train_generator.n / train_generator.batch_size) val_steps = ceil(val_generator.n / val_generator.batch_size) history = model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=cfg['TRAIN']['EPOCHS'], validation_data=val_generator, validation_steps=val_steps, callbacks=callbacks, verbose=verbose, class_weight=class_weight) # Run the model on the test set and print the resulting performance metrics. test_results = model.evaluate_generator(test_generator, verbose=1) test_metrics = {} test_summary_str = [['**Metric**', '**Value**']] for metric, value in zip(model.metrics_names, test_results): test_metrics[metric] = value print(metric, ' = ', value) test_summary_str.append([metric, str(value)]) return model, test_metrics, test_generator
outputs = {'issue': issue, 'product': product} # And combine it all in a model object model = Model(inputs=inputs, outputs=outputs, name='BERT_MultiLabel_MultiClass') # Take a look at the model model.summary() optimizer = Adam(learning_rate=5e-05, epsilon=1e-08, decay=0.01, clipnorm=1.0) # Set loss and metrics loss = { 'issue': CategoricalCrossentropy(from_logits=True), 'product': CategoricalCrossentropy(from_logits=True) } metric = { 'issue': CategoricalAccuracy('accuracy'), 'product': CategoricalAccuracy('accuracy') } # Compile the model model.compile(optimizer=optimizer, loss=loss, metrics=metric) # Ready output data for the model y_issue = to_categorical(data['Issue']) y_product = to_categorical(data['Product']) # Tokenize the input (takes some time) x = tokenizer(text=data['Consumer complaint narrative'].to_list(), add_special_tokens=True, max_length=max_length, truncation=True, padding=True, return_tensors='tf', return_token_type_ids=False,
def main(): args = cmd_parser() model_name = args.model_name # Check inputs resnetxx = ["ResNet18", "ResNet34", "ResNet50", "ResNet101", "ResNet152"] available_models = ["LeNet5", "AttentionLeNet5", "LeCunLeNet5"] + resnetxx if args.model_name not in available_models: raise ValueError( f"""args.model_name {args.model_name} NOT in {available_models}""") if args.attention: if args.attention_type == "senet": model_name = "AttentionLeNet5_SeNet" elif args.attention_type == "official": model_name = "AttentionLeNet5_Official" # Config paths date_time = datetime.now().strftime("%Y%m%d-%H%M%S") prefix = os.path.join("~", "Documents", "DeepLearningData", "mnist") # Prepare data dataset = "mnist" (train_images, train_labels), (test_images, test_labels) = load_data(dataset=dataset, if_categorical=True, if_expand_dims=True, if_normalized=False) input_shape = train_images.shape[1:] num_classes = train_labels.shape[1] # Setup model if model_name not in resnetxx: model = create_model(model_name, input_shape=input_shape, num_classes=num_classes) optimizer = create_optimizer("Adam", learning_rate=0.001) # Preprocessing and choose optimizer for ResNet18 elif model_name in resnetxx: model_core = create_model(model_name, input_shape=(32, 32, 1), num_classes=num_classes) input_ = tf.keras.layers.Input(input_shape, dtype=tf.uint8) x = tf.cast(input_, tf.float32) # padding 28x28 to 32x32 x = tf.pad(x, paddings=[[0, 0], [2, 2], [2, 2], [0, 0]]) x = model_core(x) model = tf.keras.Model(inputs=[input_], outputs=[x]) optimizer = tf.keras.optimizers.Adam( learning_rate=polynomial_schedule(0)) subfix = os.path.join(model_name, date_time) ckpt_dir = os.path.expanduser(os.path.join(prefix, subfix, "ckpts")) log_dir = os.path.expanduser(os.path.join(prefix, subfix, "logs")) os.makedirs(ckpt_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True) loss = tf.keras.losses.CategoricalCrossentropy( name="categorical_crossentropy") from tensorflow.keras.metrics import BinaryAccuracy, CategoricalAccuracy metrics = [ BinaryAccuracy(name="binary_accuracy"), CategoricalAccuracy(name="categorical_accuracy") ] model.compile(loss=loss, optimizer=optimizer, metrics=metrics) # Define callbacks from tensorflow.keras.callbacks import CSVLogger, LearningRateScheduler, TensorBoard, ModelCheckpoint lr_scheduler = LearningRateScheduler(polynomial_schedule, verbose=1) csv_logger = CSVLogger(os.path.join(log_dir, "training.log.csv"), append=True) tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir, histogram_freq=1, update_freq="batch") # without .h5 extension ckpt_filename = "%s-epoch-{epoch:03d}-categorical_accuracy-{categorical_accuracy:.4f}" % model_name ckpt_filepath = os.path.join(ckpt_dir, ckpt_filename) checkpoint_callback = ModelCheckpoint(filepath=ckpt_filepath, monitor="categorical_accuracy", verbose=1, save_weights_only=True) callbacks = [ csv_logger, lr_scheduler, checkpoint_callback, tensorboard_callback ] # Fit model epochs = 3 if if_fast_run else training_epochs model.fit(train_images, train_labels, validation_data=(test_images, test_labels), epochs=epochs, batch_size=batch_size, callbacks=callbacks)
def objective(trial): standard_object = EvaluateMNIST( train_validate_images=train_images, train_validate_labels=train_labels, validation_data_proportion=(1-JUN_SHAO_TRAINING_PROPORTION), early_stopping_significant_delta=EARLY_STOPPING_SIGNIFICANT_DELTA, early_stopping_patience=EARLY_STOPPING_PATIENCE_PARAMETER, verbosity=VERBOSITY_LEVEL_FOR_TENSORFLOW, seed=0, ) # Generate hyper-parameters standard_object.batch_size_base_two_logarithm = trial.suggest_int( 'batch_size_base_two_logarithm', 0, MAXIMUM_BATCH_SIZE_POWER_OF_TWO, ) standard_object.adam_learning_rate = trial.suggest_uniform( 'adam_learning_rate', 0, 2, ) standard_object.adam_beta_1 = trial.suggest_uniform( 'adam_beta_1', 0, 1, ) standard_object.adam_beta_2 = trial.suggest_uniform( 'adam_beta_2', 0, 1, ) standard_object.adam_epsilon = trial.suggest_uniform( 'adam_epsilon', 0, 1 ) print('\nThis trial will use the following hyper-parameters:') for key, val in trial.params.items(): print('{}: {}'.format(key, val)) if trial.number > 0: print('in an attempt to improve on best objective function score of {:0.5f}\n'.format(study.best_trial.value)) print('\n\n') # Add early stopping callback standard_object.append_early_stopper_callback() # Train and validate using hyper-parameters generated above clear_session() classifier_model = models.Sequential() classifier_model.add(layers.Conv2D( 8, (2, 2), activation='relu', input_shape=(28, 28, 1), )) classifier_model.add(layers.MaxPooling2D((2, 2), strides=2)) classifier_model.add(layers.Conv2D( 8, (2, 2), activation='relu', )) classifier_model.add(layers.MaxPooling2D((2, 2), strides=2)) classifier_model.add(layers.Flatten()) classifier_model.add(layers.Dense(10, activation='softmax')) standard_object.optimizer = AdaBeliefOptimizer( learning_rate=standard_object.adam_learning_rate, beta_1=standard_object.adam_beta_1, beta_2=standard_object.adam_beta_2, epsilon=standard_object.adam_epsilon, rectify=False, # recommended by developer amsgrad=False, # this was just another attempt to make Adam converge ) classifier_model.compile( optimizer=standard_object.optimizer, loss=CategoricalCrossentropy(), metrics=[CategoricalAccuracy()], ) standard_object.set_batch_size(standard_object.batch_size_base_two_logarithm) standard_object.stratified_split_for_training_and_validation() set_seed(standard_object.seed) classifier_model.fit( standard_object.train_split_images, standard_object.train_split_labels, epochs=MAXIMUM_EPOCHS_TO_TRAIN, validation_data=standard_object.validate_split_data, verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW, callbacks=standard_object.callbacks, batch_size=standard_object.batch_size, ) # Evaluate performance on test data and report score test_metrics = classifier_model.evaluate( test_images, test_labels, batch_size=standard_object.batch_size, ) test_results_dict = {out: test_metrics[i] for i, out in enumerate(classifier_model.metrics_names)} return(test_results_dict['categorical_accuracy'])