def get_model(max_words, maxlen, emb_dim=8, output=4):
"""
Returns multiclass classification model
"""
model = models.Sequential([
layers.Embedding(max_words, emb_dim, input_length=maxlen),
layers.Flatten(),
layers.Dense(output, activation='softmax') # 4 probability output
])
metrics = [
CategoricalAccuracy(),
Precision(),
Recall(),
AUC()
]
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy', metrics=metrics)
return model
def init_metrics(paramDict):
def jaccard_metric():
def fn(y_true, y_pred):
return jaccard(y_true, y_pred)
fn.__name__ = 'jaccard'
return fn
def dice_metric():
def fn(y_true, y_pred):
return dice(y_true, y_pred)
fn.__name__ = 'dice'
return fn
metrics = ['mse', CategoricalAccuracy(), jaccard_metric(), dice_metric()]
return metrics
def __init__(self):
self.model = Sequential()
self.model.add(Flatten(input_shape=(32, 32, 3)))
self.model.add(Dense(units=256, activation="relu"))
self.model.add(Dropout(0.10))
self.model.add(Dense(units=128, activation="relu"))
self.model.add(Dropout(0.10))
self.model.add(Dense(units=10, activation="softmax"))
self.model.compile(optimizer=RMSprop(learning_rate=0.0001),
loss=CategoricalCrossentropy(),
metrics=[
Precision(name="precision"),
Recall(name="recall"),
CategoricalAccuracy(name="accuracy"),
AUC(name="auc")
])
def RNN(x_train, y_train):
model = Sequential()
model.add(
Embedding(max_features, 256, embeddings_initializer='lecun_normal'))
model.add(SpatialDropout1D(rate=0.4))
model.add(Bidirectional(LSTM(units=64, dropout=0.4,
recurrent_dropout=0.4)))
model.add(
Dense(units=y_train.shape[1],
activation='softmax',
kernel_initializer='lecun_normal'))
METRICS = [
CategoricalAccuracy(name='accuracy'),
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
optimizer = Adam(lr=1e-5)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=METRICS)
model.summary()
earlystop = EarlyStopping(monitor='val_loss',
patience=15,
restore_best_weights=True)
reduceLR = ReduceLROnPlateau(monitor='val_loss',
factor=np.sqrt(1e-1),
verbose=1,
patience=5)
history = model.fit(x=x_train,
y=y_train,
batch_size=32,
epochs=10000,
verbose=0,
callbacks=[earlystop, reduceLR],
validation_split=0.3,
shuffle=True,
workers=4)
return model
def do_training(initial_learning_rate=0.1):
gids = get_gids_from_database("unet")
training_gen, validation_gen = initialize_train_and_validation_generators(
"unet", gids, batch_size=4, label_target_size=388)
steps_per_epoch = next(training_gen)
validation_steps = next(validation_gen)
model = UNet(input_size=(572, 572, 3))
metrics = [
Accuracy(),
CategoricalAccuracy(),
CategoricalCrossentropy(),
ArgmaxMeanIoU(num_classes=6, name="mean_iou")
]
optimizer = SGD(learning_rate=initial_learning_rate,
momentum=0.99,
nesterov=True)
model.compile(optimizer=optimizer,
loss=categorical_crossentropy,
metrics=metrics)
start_time = int(time.time())
os.mkdir(f"weights/{start_time}_{model.name}/")
metrics_to_log = [
"loss", "accuracy", "categorical_accuracy", "mean_iou",
"categorical_crossentropy"
]
model_path = f"weights/{start_time}_{model.name}/"
callbacks = [
save_model_on_epoch_end(model.name, model, model_path),
metrics_to_csv_logger(model_path + "/batch.csv", metrics_to_log),
CSVLogger(model_path + "/epoch.csv", separator=";"),
LearningRateScheduler(lr_schedule(initial_lr=initial_learning_rate)),
]
model.fit(training_gen,
epochs=20,
steps_per_epoch=steps_per_epoch,
validation_data=validation_gen,
validation_steps=validation_steps,
callbacks=callbacks)
def build_classifier(pretrained_model, num_labels, optimizer, options):
seq_len = options.seq_len
input_ids = Input(
shape=(seq_len,), dtype='int32', name='input_ids')
# token_type_ids = Input(
# shape=(seq_len,), dtype='int32', name='token_type_ids')
attention_mask = Input(
shape=(seq_len,), dtype='int32', name='attention_mask')
# inputs = [input_ids, attention_mask, token_type_ids]
inputs = [input_ids, attention_mask]
pretrained_outputs = pretrained_model(inputs)
#pooled_output = pretrained_outputs[1]
pooled_output = pretrained_outputs['last_hidden_state'][:,0,:] #CLS
# TODO consider Dropout here
if options.multiclass:
output = Dense(num_labels, activation='softmax')(pooled_output)
loss = CategoricalCrossentropy()
metrics = [CategoricalAccuracy(name='acc')]
else:
output = Dense(num_labels, activation='sigmoid')(pooled_output)
loss = BinaryCrossentropy()
metrics = [
#F1Score(name='f1_th0.3', num_classes=num_labels, average='micro', threshold=0.3),
F1Score(name='f1_th0.4', num_classes=num_labels, average='micro', threshold=0.4)#,
#F1Score(name='f1_th0.5', num_classes=num_labels, average='micro', threshold=0.5),
#AUC(name='auc', multi_label=True)
]
#output = pretrained_outputs # test
model = Model(
inputs=inputs,
outputs=[output]
)
model.compile(
optimizer=optimizer,
loss=loss,
metrics=metrics
)
return model
def build_model(max_seq_length):
in_id = tf.keras.layers.Input(shape=(max_seq_length,), name="input_ids")
in_mask = tf.keras.layers.Input(shape=(max_seq_length,), name="input_masks")
in_segment = tf.keras.layers.Input(shape=(max_seq_length,), name="segment_ids")
bert_inputs = [in_id, in_mask, in_segment]
bert_output = BertLayer(n_fine_tune_layers=10, pooling="first")(bert_inputs)
dense = tf.keras.layers.Dense(512, activation='relu')(bert_output)
pred = tf.keras.layers.Dense(12, activation='softmax')(dense)
model = tf.keras.models.Model(inputs=bert_inputs, outputs=pred)
optimizer = Adam(learning_rate=5e-05,epsilon=1e-08,decay=0.01,clipnorm=1.0)
loss = CategoricalCrossentropy(from_logits = True)
metric = CategoricalAccuracy('accuracy')
model.compile(loss=loss, optimizer=optimizer, metrics=[metric])#['accuracy'])
model.summary()
return model
def define_model(self):
inputs = tf.keras.Input(shape=(n_inputs, 1), name='input')
# 64 filters, 10 kernel size
x = Conv1D(64, 10, activation='relu')(inputs)
x = MaxPool1D()(x)
x = BatchNormalization()(x)
x = Conv1D(128, 10, activation='relu')(x)
x = MaxPool1D()(x)
x = BatchNormalization()(x)
x = Conv1D(128, 10, activation='relu')(x)
x = MaxPool1D()(x)
x = BatchNormalization()(x)
x = Conv1D(256, 10, activation='relu')(x)
x = MaxPool1D()(x)
x = BatchNormalization()(x)
x = Flatten()(x)
x = Dense(1024, activation='relu', name='dense_1')(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
x = Dense(2048, activation='relu', name='dense_2')(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
outputs = Dense(n_classes, activation='softmax', name='predictions')(x)
self.cnn_model = tf.keras.Model(inputs=inputs, outputs=outputs)
optimizer = tf.keras.optimizers.Adam(lr=learning_rate)
accuracy = CategoricalAccuracy()
self.cnn_model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=[accuracy])
def do_training(initial_learning_rate=0.001):
gids = get_gids_from_database("wnet")
training_gen, validation_gen = initialize_train_and_validation_generators(
"wnet",
gids,
batch_size=5,
label_target_size=256,
use_image_as_label=True)
steps_per_epoch = next(training_gen)
validation_steps = next(validation_gen)
full_model, encoder_model = WNet(nb_classes=6)
metrics = [Accuracy(), CategoricalAccuracy(), CategoricalCrossentropy()]
optimizer = Adam(lr=initial_learning_rate)
full_model.compile(optimizer=optimizer,
loss=categorical_crossentropy,
metrics=metrics)
start_time = int(time.time())
model_path = f"weights/{start_time}_{full_model.name}/"
os.mkdir(model_path)
metrics_to_log = [metric.name for metric in metrics]
callbacks = [
save_model_on_epoch_end(full_model.name, full_model, model_path),
save_model_on_epoch_end(encoder_model.name, encoder_model, model_path),
metrics_to_csv_logger(model_path + "/batch.csv",
["loss"] + metrics_to_log),
CSVLogger(model_path + "/epoch.csv", separator=";"),
]
full_model.fit(training_gen,
epochs=1,
steps_per_epoch=steps_per_epoch,
validation_data=validation_gen,
validation_steps=validation_steps,
callbacks=callbacks)
from tensorflow.keras.losses import CategoricalCrossentropy
from tensorflow.keras.metrics import Mean, CategoricalAccuracy
from user_profile_prediction.etl.embedding import Embedding
from user_profile_prediction.etl.preprocess_train_data import PreprocessTrainingData
from user_profile_prediction.training.deep_learning_model_train_step import DeepLearningModelTraining
p: PreprocessTrainingData = PreprocessTrainingData(
"/Volumes/Samsung_T5/Files/Document/china_hadoop/GroupProject/project_data/data/train.csv",
embedding_size=500,
sentence_len=400)
p.split_sentence()
e = Embedding(500, 10)
m = e.train_word2vec_model(p.sentences_with_split_words)
# m = e.load_embedding_model()
x_train, x_val, y_train, y_val = p.split_data(p.age_data_iter(e))
text_rnn = TextRNN(400, 500, 6)
optimizer: Adam = Adam(learning_rate=1e-4)
losses: CategoricalCrossentropy = CategoricalCrossentropy()
metric = CategoricalAccuracy()
step = DeepLearningModelTraining(text_rnn, e, optimizer, losses, metric)
step.build((None, 400, 500))
step.compile()
step.fit(x_train, y_train, x_val, y_val, 500, 50)
def train(self, x_train_main, x_train_aux, y_train,
x_val, y_val,
auxdt_size,
num_layers_intnet,
batch_size,
epochs_prior, epochs_perc, epochs_adj,
decay,
history):
# 1：事前学習
x_train = np.concatenate([x_train_main, x_train_aux], axis=1)
self.model_wholenet.summary()
self.model_wholenet.fit(x_train, y_train,
epochs=epochs_prior,
batch_size=batch_size,
verbose=self.verbose,
validation_data=(x_val, y_val),
callbacks=[history])
# 2：浸透学習
epoch = 0
loss = 1
non_perc_rate = 1 # 非浸透率の初期値
nprate_min = 1e-8 # 非浸透率の閾値
loss_min = 1e-5 # 損失関数の値の閾値
# 統合サブネットの重みを固定
for i in range(3 * (num_layers_intnet - 1) + 2):
self.model_wholenet.layers[-i-1].trainable = False
self.model_wholenet.compile(optimizer=self.optimizer, loss=categorical_crossentropy,
metrics=[CategoricalAccuracy(), Precision(), Recall()])
# 浸透学習
self.model_wholenet.summary()
while epoch < epochs_perc:
non_perc_rate = (1 - decay) ** epoch
x_train[:, -auxdt_size:] *= (1 - decay)
print('Non-Percolation Rate =', non_perc_rate)
self.model_wholenet.fit(x_train, y_train,
initial_epoch=epochs_prior + epoch, epochs=epochs_prior + epoch + 1,
batch_size=batch_size,
verbose=self.verbose,
validation_data=(x_val, y_val),
callbacks=[history])
epoch += 1
# 3：微調整
# 統合サブネットの重み固定を解除
for i in range(3 * (num_layers_intnet - 1) + 2):
self.model_wholenet.layers[-i-1].trainable = True
self.model_wholenet.compile(optimizer=self.optimizer, loss=categorical_crossentropy,
metrics=[CategoricalAccuracy(), Precision(), Recall()])
self.model_wholenet.summary()
if True: # 微調整を行う条件を入れる(現状は常に微調整を行う)
non_perc_rate = 0
x_train_aux *= non_perc_rate
x_train = np.concatenate([x_train_main, x_train_aux], axis=1)
self.model_wholenet.fit(x_train, y_train,
initial_epoch=epochs_prior + epochs_perc,
epochs=epochs_prior + epoch + epochs_adj,
batch_size=batch_size,
verbose=self.verbose,
validation_data=(x_val, y_val),
callbacks=[history])
return history
def objective(trial: optuna.Trial):
base_lr = trial.suggest_float("base_lr", 0.001, 0.05, step=0.001)
weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1e-3)
ema = trial.suggest_categorical("ema", ["true", "false"])
ema_decay = trial.suggest_loguniform("ema_decay", 0.99,
0.9999) if ema == 'true' else None
@curry
def transform(image, label, training):
image = pad(image, 2)
image, label = to_tensor(image, label)
image = normalize(image, [0.1307], [0.3081])
label = tf.one_hot(label, 10)
return image, label
batch_size = 128
eval_batch_size = 256
ds_train, ds_test, steps_per_epoch, test_steps = make_mnist_dataset(
batch_size, eval_batch_size, transform, sub_ratio=0.01)
model = LeNet5()
model.build((None, 32, 32, 1))
criterion = CrossEntropy()
epochs = 20
lr_shcedule = CosineLR(base_lr, steps_per_epoch, epochs=epochs, min_lr=0)
optimizer = SGD(lr_shcedule,
momentum=0.9,
nesterov=True,
weight_decay=weight_decay)
train_metrics = {
'loss': Mean(),
'acc': CategoricalAccuracy(),
}
eval_metrics = {
'loss': CategoricalCrossentropy(from_logits=True),
'acc': CategoricalAccuracy(),
}
learner = SuperLearner(model,
criterion,
optimizer,
train_metrics=train_metrics,
eval_metrics=eval_metrics,
work_dir=f"./MNIST",
multiple_steps=True)
callbacks = [OptunaReportIntermediateResult('acc', trial)]
# if ema == 'true':
# callbacks.append(EMA(ema_decay))
learner.fit(ds_train,
epochs,
ds_test,
val_freq=2,
steps_per_epoch=steps_per_epoch,
val_steps=test_steps,
callbacks=callbacks)
return learner.metric_history.get_metric('acc', "eval")[-1]
class ExperimentClassify:
def __init__(self, model, optimizer, exptConfig):
self.now = dt.now().strftime('%Y-%m-%d--%H-%M-%S')
self.exptConfig = exptConfig
self.model = model
self.optimizer = optimizer
self.loss = CategoricalCrossentropy(
from_logits=exptConfig['LossParams']['fromLogits'])
# ------------ metrics ----------------------
self.catAccTest = CategoricalAccuracy()
self.catAccTrain = CategoricalAccuracy()
self.exptFolder = os.path.join(
exptConfig['OtherParams']['exptBaseFolder'], self.now,
exptConfig['ModelParams']['name'])
self.modelFolder = os.path.join(self.exptFolder, 'model')
self.chkptFolder = os.path.join(self.exptFolder, 'checkpoints')
os.makedirs(self.modelFolder, exist_ok=True)
os.makedirs(self.chkptFolder, exist_ok=True)
self.stepNumber = 0
self.evalNumber = 0
self.epoch = 0
# All the logs go here ...
# ------------------------
self.createMetaData()
self.logDir = os.path.join(self.exptFolder, 'logs')
self.scalarWriter = tf.summary.create_file_writer(
os.path.join(self.logDir, 'scalars', 'metrics'))
self.graphWriter = tf.summary.create_file_writer(
os.path.join(self.logDir, 'graph'))
return
def step(self, x, y):
with tf.GradientTape() as tape:
yHat = self.model.call(x)
loss = self.loss(y, yHat)
grads = tape.gradient(loss, self.model.trainable_weights)
self.optimizer.apply_gradients(
zip(grads, self.model.trainable_weights))
self.catAccTrain.update_state(y, yHat)
with self.scalarWriter.as_default():
tf.summary.scalar('training loss', data=loss, step=self.stepNumber)
tf.summary.scalar('training accuracy',
data=self.catAccTrain.result().numpy(),
step=self.stepNumber)
self.stepNumber += 1
return loss.numpy()
def eval(self, x, y):
yHat = self.model.predict(x)
self.catAccTest.update_state(y, yHat)
with self.scalarWriter.as_default():
tf.summary.scalar('testing accuracy',
data=self.catAccTest.result().numpy(),
step=self.evalNumber)
self.evalNumber += 1
return self.catAccTest.result().numpy()
def createMetaData(self):
if not os.path.exists(self.exptFolder):
os.makedirs(self.exptFolder)
with open(os.path.join(self.exptFolder, 'config.json'), 'w') as fOut:
json.dump(self.exptConfig, fOut)
return
def createModelSummary(self, x):
tf.summary.trace_on(graph=True)
self.model.predict(x)
with self.graphWriter.as_default():
tf.summary.trace_export('name', step=0)
tf.summary.trace_off()
def saveModel(self):
try:
self.model.save(self.modelFolder)
except Exception as e:
print(f'Unable to save the model: {e}')
return
def checkPoint(self):
try:
epoch = self.epoch
step = self.stepNumber
self.model.save_weights(
os.path.join(self.chkptFolder, f'{epoch:07d}-{step:07d}'))
except Exception as e:
print(f'Unable to checkpoint: {self.stepNumber}: {e}')
return
def training_model(training_file_path: str, embedding_size: int,
sentence_len: int, conv_filter: int, global_max_pool: bool,
pool_size: int, drop_rate: float, dense_size: int,
l1_regularization: float, l2_regularization: float,
vocabulary_size: int, min_count: int, class_num: int,
label_name: int, learning_rate: int, epochs: int,
batch_size: int, checkpoint_saved_path: str):
global TRAINING_FILE_PATH, EMBEDDING_SIZE, SENTENCE_LEN, MIN_COUNT, CLASS_NUM, LEARNING_RATE,\
EPOCHS, BATCH_SIZE, MODEL, VOCABULARY_SIZE
TRAINING_FILE_PATH = training_file_path
EMBEDDING_SIZE = embedding_size
SENTENCE_LEN = sentence_len
VOCABULARY_SIZE = vocabulary_size
MIN_COUNT = min_count
CLASS_NUM = class_num
LEARNING_RATE = learning_rate
EPOCHS = epochs
BATCH_SIZE = batch_size
preprocess: PreprocessTrainingData = PreprocessTrainingData(
csv_file_path=TRAINING_FILE_PATH,
embedding_size=EMBEDDING_SIZE,
sentence_len=SENTENCE_LEN,
vocabulary_size=VOCABULARY_SIZE)
preprocess.split_sentence()
tokenizer = preprocess.tokenize()
x_train, x_val, y_train, y_val = preprocess.split_data(
preprocess.age_data_iter())
text_cnn = TextCNN(SENTENCE_LEN, EMBEDDING_SIZE, CLASS_NUM,
VOCABULARY_SIZE, conv_filter, global_max_pool,
pool_size, drop_rate, dense_size, l1_regularization,
l2_regularization)
optimizer: Adam = Adam(learning_rate=LEARNING_RATE)
losses: CategoricalCrossentropy = CategoricalCrossentropy()
metric = CategoricalAccuracy()
training = DeepLearningModelTraining(text_cnn, optimizer, losses, metric)
training.build((
None,
SENTENCE_LEN,
))
training.compile()
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)
training.fit(x_train,
y_train,
x_val,
y_val,
epochs=EPOCHS,
batch=BATCH_SIZE,
callbacks=[tensorboard_callback])
ckpt = tf.train.Checkpoint(text_cnn=training.training_model)
ckpt_manager = tf.train.CheckpointManager(ckpt,
checkpoint_saved_path,
max_to_keep=5)
path = ckpt_manager.save()
print("checkpoint of text_cnn has been saved in {}".format(path))
inputs = Input(shape=input_shape)
teacher = KDModel.Teacher(NUM_CLASSES)
teacher_model = teacher.createModel(inputs)
# Teacherモデルの学習
optimizer = Adamax(learning_rate=LR_T) # 最適化アルゴリズム
history_teacher = LossAccHistory()
training = KDModel.NormalTraining(teacher_model)
teacher_model.summary()
# plot_model(teacher_model, show_shapes=True, to_file='teacher_model.png')
for epoch in range(1, EPOCHS_T + 1):
loss_metric = Mean()
loss_metric_val = Mean()
acc_metric = Mean()
acc_metric_val = Mean()
acc = CategoricalAccuracy()
acc_val = CategoricalAccuracy()
# 各バッチごとに学習
for (x_train_, y_train_), (x_val_, y_val_) in ds:
loss_value, grads = training.grad(x_train_, y_train_)
optimizer.apply_gradients(zip(grads, teacher_model.trainable_weights))
loss_value_test = training.loss(x_val_, y_val_)
probs = tf.nn.softmax(teacher_model(x_train_))
probs_val = tf.nn.softmax(teacher_model(x_val_))
loss_metric(loss_value)
acc_metric(acc(y_train_, probs))
loss_metric_val(loss_value_test)
acc_metric_val(acc_val(y_val_, probs_val))
### ------- Train the model ------- ###
# Set an optimizer
optimizer = Adam(learning_rate=5e-05, epsilon=1e-08, decay=0.01, clipnorm=1.0)
# Set loss and metrics
loss = {
'gia': CategoricalCrossentropy(from_logits=True),
'dich_vu': CategoricalCrossentropy(from_logits=True),
'an_toan': CategoricalCrossentropy(from_logits=True),
'chat_luong': CategoricalCrossentropy(from_logits=True),
'ship': CategoricalCrossentropy(from_logits=True),
'chinh_hang': CategoricalCrossentropy(from_logits=True)
}
metric = {
'gia': CategoricalAccuracy('accuracy'),
'dich_vu': CategoricalAccuracy('accuracy'),
'an_toan': CategoricalAccuracy('accuracy'),
'chat_luong': CategoricalAccuracy('accuracy'),
'ship': CategoricalAccuracy('accuracy'),
'chinh_hang': CategoricalAccuracy('accuracy')
}
# Compile the model
model.compile(optimizer=optimizer, loss=loss, metrics=metric)
# Ready output data for the model
y_gia = to_categorical(data['giá'])
y_dich_vu = to_categorical(data['dịch_vụ'])
y_an_toan = to_categorical(data['an_toàn'])
y_chat_luong = to_categorical(data['chất_lượng'])
# train, val, eval 데이터셋으로 변환
train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(10000).batch(32)
val_ds = tf.data.Dataset.from_tensor_slices((X_val, y_val)).batch(32)
# model 생성
model = MyModel(config)
# loss 함수와 옵티마이저 설정
loss_object = CategoricalCrossentropy() # y label이 one-hot encoding
optimizer = Adam()
# train, val의 loss와 accuracy 설정
train_loss = Mean(name='train_loss')
train_accuracy = CategoricalAccuracy(name='train_accuracy')
val_loss = Mean(name='val_loss')
val_accuracy = CategoricalAccuracy(name='val_accuracy')
########################################################################
## train
########################################################################
print('\n\n*** 학습을 시작합니다...')
ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model)
manager = tf.train.CheckpointManager(checkpoint=ckpt, directory='./tf_ckpts',
max_to_keep=config['epochs'])
# Then build your model output
author = Dense(
units=len(data.author.value_counts()),
kernel_initializer=TruncatedNormal(stddev=config.initializer_range),
name='author')(pooled_output)
outputs = {'author': author}
# And combine it all in a model object
model = Model(inputs=inputs, outputs=outputs, name='BERT_MultiClass')
# Take a look at the model
model.summary()
# Set an optimizer
optimizer = Adam(learning_rate=5e-05, epsilon=1e-08, decay=0.01, clipnorm=1.0)
# Set loss and metrics
loss = {'author': CategoricalCrossentropy(from_logits=True)}
metric = {'author': CategoricalAccuracy('accuracy')}
# Compile the model
model.compile(optimizer=optimizer, loss=loss, metrics=metric)
# Ready output data for the model
y_author = to_categorical(data['author'])
# Tokenize the input (takes some time)
x = tokenizer(text=data['text_without_stopwords'].to_list(),
add_special_tokens=True,
max_length=max_length,
truncation=True,
padding=True,
return_tensors='tf',
return_token_type_ids=False,
return_attention_mask=False,
verbose=True)
model.summary()
# -----------------------------------------------------------------#
# Train the model
# Set an optimizer
optimizer = Adam(
# learning_rate=5e-05,
learning_rate=1e-5,
epsilon=1e-08,
decay=0.01,
clipnorm=1.0)
# Set loss and metrics
loss = {'classification': CategoricalCrossentropy(from_logits=True)}
metric = {'classification': CategoricalAccuracy('accuracy')}
# Compile the model
model.compile(optimizer=optimizer, loss=loss, metrics=metric)
# Ready output data for the model
y_classification = to_categorical(data['Classification'])
# Tokenize the input (takes some time)
x = tokenizer(text=data['Comments'].to_list(),
add_special_tokens=True,
max_length=max_length,
truncation=True,
padding=True,
return_tensors='tf',
return_token_type_ids=False,
training_iterator = data_generator.flow(x_train, y_train, batch_size=Batch_size) validation_iterator = data_generator.flow(x_valid, y_valid, batch_size=Batch_size) # Build the model model = Sequential() model.add(Input(shape=(128, 128, 3))) #refer to input_data.shape # Adding two convolutional layers, interspersed with max pooling layers, followed by two dense layers: model.add(Conv2D(8, 3, strides=2, activation='relu')) model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2))) model.add(Conv2D(8, 3, strides=2, activation='relu')) model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2))) model.add(Flatten()) model.add(Dense(16, activation='relu')) # Output model.add(Dense(4, activation='softmax')) model.compile(optimizer=Adam(learning_rate=0.001), loss=CategoricalCrossentropy(), metrics=[CategoricalAccuracy(),AUC()]) # Because the labels are one-hot categories, use tf.keras.losses.CategoricalCrossentropy() as your loss. print(model.summary()) # Train model.fit(training_iterator, steps_per_epoch=x_train.shape[0]/Batch_size, epochs=8, validation_data=validation_iterator, validation_steps=y_train.shape[0]/Batch_size) #can be len(x_train) instead # model’s accuracy should be around 0.60-0.70, and your AUC should fall into the 0.80-0.90 range: val_categorical_accuracy: 0.6152 - val_auc: 0.8441 # Your accuracy tells you that your model assigns the highest probability to the correct class more than 60% of the time. For a classification task with over four classes, this is no small feat: a random baseline model would achieve only ~25% accuracy on the dataset. Your AUC tells you that for a random galaxy, there is more than an 80% chance your model would assign a higher probability to a true class than to a false one. #If you would like, try tweaking your architecture. Can you find a better set of hyperparameters? MAKE SURE to watch your parameter count: it’s easy to accidentally create a model with more than tens of thousands of parameters, which could overfit to your relatively small dataset (or crash the Learning Environment). Note that scores will fluctuate a bit, depending on how the weights are randomly initialized. # learning rate # number of convolutional layers # number of filters, strides, and padding type per layer # stride and pool_size of max pooling layers # size of hidden linear layers
def train_network(model, train_x, train_y, val_x, val_y, batch_s, epochs1,
epochs2):
"""
Function to train network in two steps:
* Train network with initial VGG base layers frozen
* Unfreeze all layers and retrain with smaller learning rate
:param model: CNN model
:param train_x: training input
:param train_y: training outputs
:param val_x: validation inputs
:param val_y: validation outputs
:param batch_s: batch size
:param epochs1: epoch count for initial training
:param epochs2: epoch count for training all layers unfrozen
:return: trained network
"""
# Freeze VGG19 pre-trained layers.
if config.imagesize == "large":
model.layers[0].layers[1].trainable = False
else:
model.layers[0].trainable = False
# Train model with frozen layers (all training with early stopping dictated by loss in validation over 3 runs).
if config.dataset == "mini-MIAS":
model.compile(optimizer=Adam(1e-3),
loss=CategoricalCrossentropy(),
metrics=[CategoricalAccuracy()])
hist_1 = model.fit(x=train_x,
y=train_y,
batch_size=batch_s,
steps_per_epoch=len(train_x) // batch_s,
validation_data=(val_x, val_y),
validation_steps=len(val_x) // batch_s,
epochs=epochs1,
callbacks=[
EarlyStopping(
monitor='val_categorical_accuracy',
patience=8,
restore_best_weights=True),
ReduceLROnPlateau(patience=4)
])
elif config.dataset == "CBIS-DDSM":
model.compile(optimizer=Adam(lr=1e-4),
loss=BinaryCrossentropy(),
metrics=[BinaryAccuracy()])
hist_1 = model.fit(x=train_x,
validation_data=val_x,
epochs=epochs1,
callbacks=[
EarlyStopping(monitor='val_loss',
patience=8,
restore_best_weights=True),
ReduceLROnPlateau(patience=6)
])
# Plot the training loss and accuracy.
plot_training_results(hist_1, "Initial_training", True)
# Train a second time with a smaller learning rate and with all layers unfrozen
# (train over fewer epochs to prevent over-fitting).
if config.imagesize == "large":
model.layers[0].layers[1].trainable = True
else:
model.layers[0].trainable = True
if config.dataset == "mini-MIAS":
model.compile(
optimizer=Adam(1e-5), # Very low learning rate
loss=CategoricalCrossentropy(),
metrics=[CategoricalAccuracy()])
hist_2 = model.fit(x=train_x,
y=train_y,
batch_size=batch_s,
steps_per_epoch=len(train_x) // batch_s,
validation_data=(val_x, val_y),
validation_steps=len(val_x) // batch_s,
epochs=epochs2,
callbacks=[
EarlyStopping(
monitor='val_categorical_accuracy',
patience=8,
restore_best_weights=True),
ReduceLROnPlateau(patience=6)
])
elif config.dataset == "CBIS-DDSM":
model.compile(
optimizer=Adam(lr=1e-5), # Very low learning rate
loss=BinaryCrossentropy(),
metrics=[BinaryAccuracy()])
hist_2 = model.fit(x=train_x,
validation_data=val_x,
epochs=epochs2,
callbacks=[
EarlyStopping(monitor='val_loss',
patience=10,
restore_best_weights=True),
ReduceLROnPlateau(patience=6)
])
# Plot the training loss and accuracy.
plot_training_results(hist_2, "Fine_tuning_training", False)
return model
def generate_compiled_segmentation_model(
model_name,
model_parameters,
num_classes,
loss,
optimizer,
weights_to_load=None,
optimizing_threshold_class_metric=None,
optimizing_class_id=None,
optimizing_input_threshold=None,
optimized_class_thresholds=None):
# These are the only model, loss, and optimizer currently supported
assert model_name == 'Unet'
assert loss == 'cross_entropy'
assert optimizer == 'adam'
loss_fn = BinaryCrossentropyL()
all_metrics = [
] # one-hot versions are generally preferred for given metric
# make first metric a copy of loss, to continually verify `val_loss` is correct
if isinstance(loss_fn, BinaryCrossentropyL):
all_metrics.append(BinaryCrossentropyM(name='binary_ce_metric'))
else:
all_metrics.append(CategoricalCrossentropyM(name='categ_ce_metric'))
# standard thresholded version (default threshold is 0.5) also kept below, in case it's desired in certain scenario
for class_num in range(num_classes + 1):
if class_num == 0 and optimizing_threshold_class_metric is None: # all class metrics
# note, `loss_fn` for all classes placed before `all_metrics` in lineup of command window metrics and plots
if not isinstance(loss_fn, BinaryCrossentropyL):
all_metrics.extend([CategoricalCELoss()])
all_metrics[1].name = str('categ_cross_entropy_sm')
all_metrics.extend([
AccuracyTfKeras(),
# OneHotAccuracyTfKeras(), # `global_threshold` built-in
ClassBinaryAccuracyTfKeras(thresholds=global_threshold),
# OneHotClassBinaryAccuracyTfKeras(thresholds=global_threshold),
ClassBinaryAccuracySM(threshold=global_threshold),
# OneHotClassBinaryAccuracySM(threshold=global_threshold),
BinaryAccuracy(threshold=global_threshold),
CategoricalAccuracy(),
FalseNegatives(name='false_neg', thresholds=global_threshold),
# OneHotFalseNegatives(name='false_neg_1H', thresholds=global_threshold),
TrueNegatives(name='true_neg', thresholds=global_threshold),
# OneHotTrueNegatives(name='true_neg_1H', thresholds=global_threshold),
FalsePositives(name='false_pos', thresholds=global_threshold),
# OneHotFalsePositives(name='false_pos_1H', thresholds=global_threshold),
TruePositives(name='true_pos', thresholds=global_threshold),
# OneHotTruePositives(name='true_pos_1H', thresholds=global_threshold),
Recall(name='recall', thresholds=global_threshold),
# OneHotRecall(name='recall_1H', thresholds=global_threshold),
Precision(name='precision', thresholds=global_threshold),
# OneHotPrecision(name='precision_1H', thresholds=global_threshold),
FBetaScore(name='f1_score',
beta=1,
thresholds=global_threshold),
# OneHotFBetaScore(name='f1_score_1H', beta=1, thresholds=global_threshold),
IoUScore(name='iou_score', thresholds=global_threshold),
# OneHotIoUScore(name='iou_score_1H', thresholds=global_threshold)
])
elif class_num == 0 and optimizing_threshold_class_metric is not None: # all class metrics
continue
else: # per class metrics
if optimizing_threshold_class_metric is not None:
class_threshold = optimizing_input_threshold
class_num = optimizing_class_id + 1
elif optimized_class_thresholds is None:
class_threshold = global_threshold
else:
class_threshold = optimized_class_thresholds[str(
'class' + str(class_num - 1))]
all_metrics.append(CategoricalCELoss(class_indexes=class_num - 1))
all_metrics[-1].name = str('class' + str(class_num - 1) +
'_binary_cross_entropy')
all_metrics.append(
ClassBinaryAccuracySM(name=str('class' + str(class_num - 1) +
'_binary_accuracy_sm'),
class_indexes=class_num - 1,
threshold=class_threshold))
all_metrics.append(
ClassBinaryAccuracyTfKeras(
name=str('class' + str(class_num - 1) +
'_binary_accuracy_tfkeras'),
class_id=class_num - 1,
thresholds=class_threshold))
all_metrics.append(
IoUScore(name=str('class' + str(class_num - 1) + '_iou_score'),
class_id=class_num - 1,
thresholds=class_threshold))
all_metrics.append(
FBetaScore(name=str('class' + str(class_num - 1) +
'_f1_score'),
class_id=class_num - 1,
beta=1,
thresholds=class_threshold))
all_metrics.append(
Precision(name=str('class' + str(class_num - 1) +
'_precision'),
class_id=class_num - 1,
thresholds=class_threshold))
all_metrics.append(
Recall(name=str('class' + str(class_num - 1) + '_recall'),
class_id=class_num - 1,
thresholds=class_threshold))
if optimizing_threshold_class_metric is not None:
break
if num_classes == 1:
break
# strategy = tf.distribute.MirroredStrategy()
# with strategy.scope():
model = Unet(input_shape=(None, None, 1),
classes=num_classes,
**model_parameters)
model.compile(optimizer=Adam(), loss=loss_fn, metrics=all_metrics)
if weights_to_load:
model.load_weights(weights_to_load)
if optimizing_threshold_class_metric is None:
print(model.summary())
return model
'/content/test1',
class_mode='categorical',
color_mode='grayscale',
batch_size=BATCH_SIZE)
model = tf.keras.Sequential()
model.add(tf.keras.Input(shape=(256, 256, 1)))
model.add(tf.keras.layers.Conv2D(8, 2, strides=2, activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(5, 5), strides=(5, 5)))
model.add(tf.keras.layers.Conv2D(8, 2, strides=2, activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(64, activation='relu'))
model.add(tf.keras.layers.Dense(3, activation='softmax'))
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=[CategoricalAccuracy(), AUC()])
model.summary()
stop = EarlyStopping(monitor='categorical_accuracy',
mode='max',
verbose=1,
patience=3)
model.fit(training_iterator,
steps_per_epoch=training_iterator.samples / BATCH_SIZE,
epochs=15,
validation_data=validation_iterator,
validation_steps=validation_iterator.samples / BATCH_SIZE,
callbacks=[stop])
to_predict = ImageDataGenerator().flow_from_directory('/content/predictcovid',
A_in = Input(shape=(None,), sparse=True)
I_in = Input(shape=(), name='segment_ids_in', dtype=tf.int32)
X_1 = GraphConvSkip(32, activation='relu')([X_in, A_in])
X_1, A_1, I_1 = TopKPool(ratio=0.5)([X_1, A_in, I_in])
X_2 = GraphConvSkip(32, activation='relu')([X_1, A_1])
X_2, A_2, I_2 = TopKPool(ratio=0.5)([X_2, A_1, I_1])
X_3 = GraphConvSkip(32, activation='relu')([X_2, A_2])
X_3 = GlobalAvgPool()([X_3, I_2])
output = Dense(n_out, activation='softmax')(X_3)
# Build model
model = Model(inputs=[X_in, A_in, I_in], outputs=output)
opt = Adam(lr=learning_rate)
loss_fn = CategoricalCrossentropy()
acc_fn = CategoricalAccuracy()
@tf.function(
input_signature=(tf.TensorSpec((None, F), dtype=tf.float64),
tf.SparseTensorSpec((None, None), dtype=tf.float32),
tf.TensorSpec((None,), dtype=tf.int32),
tf.TensorSpec((None, n_out), dtype=tf.float64)),
experimental_relax_shapes=True)
def train_step(X_, A_, I_, y_):
with tf.GradientTape() as tape:
predictions = model([X_, A_, I_], training=True)
loss = loss_fn(y_, predictions)
loss += sum(model.losses)
acc = acc_fn(y_, predictions)
gradients = tape.gradient(loss, model.trainable_variables)
ds = tf.data.Dataset.zip((ds_train, ds_val))
# Teacherモデルの定義
inputs_main = Input(shape=input_shape_main)
inputs_aux = Input(shape=input_shape_aux)
teacher = KDModel.Teacher(NUM_CLASSES, T)
teacher_model = teacher.createModel(inputs_main, inputs_aux)
# Teacherモデルの学習
training = KDModel.NormalTraining(teacher_model)
teacher_model.summary()
# plot_model(teacher_model, show_shapes=True, to_file='teacher_model.png')
for epoch in range(1, EPOCHS_T + 1):
epoch_loss_avg = Mean()
epoch_loss_avg_val = Mean()
epoch_accuracy = CategoricalAccuracy()
epoch_accuracy_val = CategoricalAccuracy()
# 各バッチごとに学習
for (x_train_main_, x_train_aux_, y_train_), (x_val_main_, x_val_aux_,
y_val_) in ds:
loss_value, grads = training.grad([x_train_main_, x_train_aux_],
y_train_)
optimizer.apply_gradients(zip(grads,
teacher_model.trainable_variables))
loss_value_test = training.loss([x_val_main_, x_val_aux_], y_val_)
epoch_loss_avg(loss_value)
epoch_accuracy(
y_train_,
tf.nn.softmax(teacher_model([x_train_main_, x_train_aux_])))
drop_path = 0.3
model = NASNet(16, 8, True, drop_path, 10, PC_DARTS_cifar)
model.build((None, 32, 32, 3))
model.summary()
criterion = CrossEntropy(auxiliary_weight=0.4)
base_lr = 0.025
epochs = 600
lr_schedule = CosineLR(base_lr * mul, steps_per_epoch, epochs=epochs, min_lr=0)
optimizer = SGD(lr_schedule, momentum=0.9, weight_decay=3e-4, nesterov=True)
train_metrics = {
'loss': Mean(),
'acc': CategoricalAccuracy(),
}
eval_metrics = {
'loss': CategoricalCrossentropy(from_logits=True),
'acc': CategoricalAccuracy(),
}
learner = SuperLearner(model,
criterion,
optimizer,
grad_clip_norm=5.0,
jit_compile=False,
train_metrics=train_metrics,
eval_metrics=eval_metrics,
work_dir="./CIFAR10-PC-DARTS")
def train_model(cfg, data, callbacks, verbose=1):
'''
Train a and evaluate model on given data.
:param cfg: Project config (from config.yml)
:param data: dict of partitioned dataset
:param callbacks: list of callbacks for Keras model
:param verbose: Verbosity mode to pass to model.fit_generator()
:return: Trained model and associated performance metrics on the test set
'''
# If set in config file, oversample the minority class
if cfg['TRAIN']['IMB_STRATEGY'] == 'random_oversample':
data['TRAIN'] = random_minority_oversample(data['TRAIN'])
# Create ImageDataGenerators
train_img_gen = ImageDataGenerator(rotation_range=10, preprocessing_function=remove_text,
samplewise_std_normalization=True, samplewise_center=True)
val_img_gen = ImageDataGenerator(preprocessing_function=remove_text,
samplewise_std_normalization=True, samplewise_center=True)
test_img_gen = ImageDataGenerator(preprocessing_function=remove_text,
samplewise_std_normalization=True, samplewise_center=True)
# Create DataFrameIterators
img_shape = tuple(cfg['DATA']['IMG_DIM'])
y_col = 'label_str'
class_mode = 'categorical'
train_generator = train_img_gen.flow_from_dataframe(dataframe=data['TRAIN'], directory=cfg['PATHS']['RAW_DATA'],
x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'],
class_mode=class_mode, validate_filenames=False)
val_generator = val_img_gen.flow_from_dataframe(dataframe=data['VAL'], directory=cfg['PATHS']['RAW_DATA'],
x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'],
class_mode=class_mode, validate_filenames=False)
test_generator = test_img_gen.flow_from_dataframe(dataframe=data['TEST'], directory=cfg['PATHS']['RAW_DATA'],
x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'],
class_mode=class_mode, validate_filenames=False, shuffle=False)
# Save model's ordering of class indices
dill.dump(test_generator.class_indices, open(cfg['PATHS']['OUTPUT_CLASS_INDICES'], 'wb'))
# Apply class imbalance strategy. We have many more X-rays negative for COVID-19 than positive.
histogram = np.bincount(np.array(train_generator.labels).astype(int)) # Get class distribution
class_weight = None
if cfg['TRAIN']['IMB_STRATEGY'] == 'class_weight':
class_multiplier = cfg['TRAIN']['CLASS_MULTIPLIER']
class_multiplier = [class_multiplier[cfg['DATA']['CLASSES'].index(c)] for c in test_generator.class_indices]
class_weight = get_class_weights(histogram, class_multiplier)
# Define metrics.
covid_class_idx = test_generator.class_indices['COVID-19'] # Get index of COVID-19 class
thresholds = 1.0 / len(cfg['DATA']['CLASSES']) # Binary classification threshold for a class
metrics = ['accuracy', CategoricalAccuracy(name='accuracy'),
Precision(name='precision', thresholds=thresholds, class_id=covid_class_idx),
Recall(name='recall', thresholds=thresholds, class_id=covid_class_idx),
AUC(name='auc'),
F1Score(name='f1score', thresholds=thresholds, class_id=covid_class_idx)]
# Define the model.
print('Training distribution: ', ['Class ' + list(test_generator.class_indices.keys())[i] + ': ' + str(histogram[i]) + '. '
for i in range(len(histogram))])
input_shape = cfg['DATA']['IMG_DIM'] + [3]
num_gpus = cfg['TRAIN']['NUM_GPUS']
if cfg['TRAIN']['MODEL_DEF'] == 'dcnn_resnet':
model_def = dcnn_resnet
elif cfg['TRAIN']['MODEL_DEF'] == 'resnet50v2':
model_def = resnet50v2
else:
model_def = resnet101v2
if cfg['TRAIN']['CLASS_MODE'] == 'binary':
histogram = np.bincount(data['TRAIN']['label'].astype(int))
output_bias = np.log([histogram[i] / (np.sum(histogram) - histogram[i]) for i in range(histogram.shape[0])])
model = model_def(cfg['NN']['DCNN_BINARY'], input_shape, metrics, 2, output_bias=output_bias, gpus=num_gpus)
else:
n_classes = len(cfg['DATA']['CLASSES'])
histogram = np.bincount(data['TRAIN']['label'].astype(int))
output_bias = np.log([histogram[i] / (np.sum(histogram) - histogram[i]) for i in range(histogram.shape[0])])
model = model_def(cfg['NN']['DCNN_MULTICLASS'], input_shape, metrics, n_classes, output_bias=output_bias,
gpus=num_gpus)
# Train the model.
steps_per_epoch = ceil(train_generator.n / train_generator.batch_size)
val_steps = ceil(val_generator.n / val_generator.batch_size)
history = model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=cfg['TRAIN']['EPOCHS'],
validation_data=val_generator, validation_steps=val_steps, callbacks=callbacks,
verbose=verbose, class_weight=class_weight)
# Run the model on the test set and print the resulting performance metrics.
test_results = model.evaluate_generator(test_generator, verbose=1)
test_metrics = {}
test_summary_str = [['**Metric**', '**Value**']]
for metric, value in zip(model.metrics_names, test_results):
test_metrics[metric] = value
print(metric, ' = ', value)
test_summary_str.append([metric, str(value)])
return model, test_metrics, test_generator
outputs = {'issue': issue, 'product': product}
# And combine it all in a model object
model = Model(inputs=inputs,
outputs=outputs,
name='BERT_MultiLabel_MultiClass')
# Take a look at the model
model.summary()
optimizer = Adam(learning_rate=5e-05, epsilon=1e-08, decay=0.01, clipnorm=1.0)
# Set loss and metrics
loss = {
'issue': CategoricalCrossentropy(from_logits=True),
'product': CategoricalCrossentropy(from_logits=True)
}
metric = {
'issue': CategoricalAccuracy('accuracy'),
'product': CategoricalAccuracy('accuracy')
}
# Compile the model
model.compile(optimizer=optimizer, loss=loss, metrics=metric)
# Ready output data for the model
y_issue = to_categorical(data['Issue'])
y_product = to_categorical(data['Product'])
# Tokenize the input (takes some time)
x = tokenizer(text=data['Consumer complaint narrative'].to_list(),
add_special_tokens=True,
max_length=max_length,
truncation=True,
padding=True,
return_tensors='tf',
return_token_type_ids=False,
def main():
args = cmd_parser()
model_name = args.model_name
# Check inputs
resnetxx = ["ResNet18", "ResNet34", "ResNet50", "ResNet101", "ResNet152"]
available_models = ["LeNet5", "AttentionLeNet5", "LeCunLeNet5"] + resnetxx
if args.model_name not in available_models:
raise ValueError(
f"""args.model_name {args.model_name} NOT in {available_models}""")
if args.attention:
if args.attention_type == "senet":
model_name = "AttentionLeNet5_SeNet"
elif args.attention_type == "official":
model_name = "AttentionLeNet5_Official"
# Config paths
date_time = datetime.now().strftime("%Y%m%d-%H%M%S")
prefix = os.path.join("~", "Documents", "DeepLearningData", "mnist")
# Prepare data
dataset = "mnist"
(train_images,
train_labels), (test_images, test_labels) = load_data(dataset=dataset,
if_categorical=True,
if_expand_dims=True,
if_normalized=False)
input_shape = train_images.shape[1:]
num_classes = train_labels.shape[1]
# Setup model
if model_name not in resnetxx:
model = create_model(model_name,
input_shape=input_shape,
num_classes=num_classes)
optimizer = create_optimizer("Adam", learning_rate=0.001)
# Preprocessing and choose optimizer for ResNet18
elif model_name in resnetxx:
model_core = create_model(model_name,
input_shape=(32, 32, 1),
num_classes=num_classes)
input_ = tf.keras.layers.Input(input_shape, dtype=tf.uint8)
x = tf.cast(input_, tf.float32)
# padding 28x28 to 32x32
x = tf.pad(x, paddings=[[0, 0], [2, 2], [2, 2], [0, 0]])
x = model_core(x)
model = tf.keras.Model(inputs=[input_], outputs=[x])
optimizer = tf.keras.optimizers.Adam(
learning_rate=polynomial_schedule(0))
subfix = os.path.join(model_name, date_time)
ckpt_dir = os.path.expanduser(os.path.join(prefix, subfix, "ckpts"))
log_dir = os.path.expanduser(os.path.join(prefix, subfix, "logs"))
os.makedirs(ckpt_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
loss = tf.keras.losses.CategoricalCrossentropy(
name="categorical_crossentropy")
from tensorflow.keras.metrics import BinaryAccuracy, CategoricalAccuracy
metrics = [
BinaryAccuracy(name="binary_accuracy"),
CategoricalAccuracy(name="categorical_accuracy")
]
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
# Define callbacks
from tensorflow.keras.callbacks import CSVLogger, LearningRateScheduler, TensorBoard, ModelCheckpoint
lr_scheduler = LearningRateScheduler(polynomial_schedule, verbose=1)
csv_logger = CSVLogger(os.path.join(log_dir, "training.log.csv"),
append=True)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir,
histogram_freq=1,
update_freq="batch")
# without .h5 extension
ckpt_filename = "%s-epoch-{epoch:03d}-categorical_accuracy-{categorical_accuracy:.4f}" % model_name
ckpt_filepath = os.path.join(ckpt_dir, ckpt_filename)
checkpoint_callback = ModelCheckpoint(filepath=ckpt_filepath,
monitor="categorical_accuracy",
verbose=1,
save_weights_only=True)
callbacks = [
csv_logger, lr_scheduler, checkpoint_callback, tensorboard_callback
]
# Fit model
epochs = 3 if if_fast_run else training_epochs
model.fit(train_images,
train_labels,
validation_data=(test_images, test_labels),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
def objective(trial):
standard_object = EvaluateMNIST(
train_validate_images=train_images,
train_validate_labels=train_labels,
validation_data_proportion=(1-JUN_SHAO_TRAINING_PROPORTION),
early_stopping_significant_delta=EARLY_STOPPING_SIGNIFICANT_DELTA,
early_stopping_patience=EARLY_STOPPING_PATIENCE_PARAMETER,
verbosity=VERBOSITY_LEVEL_FOR_TENSORFLOW,
seed=0,
)
# Generate hyper-parameters
standard_object.batch_size_base_two_logarithm = trial.suggest_int(
'batch_size_base_two_logarithm',
0,
MAXIMUM_BATCH_SIZE_POWER_OF_TWO,
)
standard_object.adam_learning_rate = trial.suggest_uniform(
'adam_learning_rate',
0,
2,
)
standard_object.adam_beta_1 = trial.suggest_uniform(
'adam_beta_1',
0,
1,
)
standard_object.adam_beta_2 = trial.suggest_uniform(
'adam_beta_2',
0,
1,
)
standard_object.adam_epsilon = trial.suggest_uniform(
'adam_epsilon',
0,
1
)
print('\nThis trial will use the following hyper-parameters:')
for key, val in trial.params.items():
print('{}: {}'.format(key, val))
if trial.number > 0:
print('in an attempt to improve on best objective function score of {:0.5f}\n'.format(study.best_trial.value))
print('\n\n')
# Add early stopping callback
standard_object.append_early_stopper_callback()
# Train and validate using hyper-parameters generated above
clear_session()
classifier_model = models.Sequential()
classifier_model.add(layers.Conv2D(
8,
(2, 2),
activation='relu',
input_shape=(28, 28, 1),
))
classifier_model.add(layers.MaxPooling2D((2, 2), strides=2))
classifier_model.add(layers.Conv2D(
8,
(2, 2),
activation='relu',
))
classifier_model.add(layers.MaxPooling2D((2, 2), strides=2))
classifier_model.add(layers.Flatten())
classifier_model.add(layers.Dense(10, activation='softmax'))
standard_object.optimizer = AdaBeliefOptimizer(
learning_rate=standard_object.adam_learning_rate,
beta_1=standard_object.adam_beta_1,
beta_2=standard_object.adam_beta_2,
epsilon=standard_object.adam_epsilon,
rectify=False, # recommended by developer
amsgrad=False, # this was just another attempt to make Adam converge
)
classifier_model.compile(
optimizer=standard_object.optimizer,
loss=CategoricalCrossentropy(),
metrics=[CategoricalAccuracy()],
)
standard_object.set_batch_size(standard_object.batch_size_base_two_logarithm)
standard_object.stratified_split_for_training_and_validation()
set_seed(standard_object.seed)
classifier_model.fit(
standard_object.train_split_images,
standard_object.train_split_labels,
epochs=MAXIMUM_EPOCHS_TO_TRAIN,
validation_data=standard_object.validate_split_data,
verbose=VERBOSITY_LEVEL_FOR_TENSORFLOW,
callbacks=standard_object.callbacks,
batch_size=standard_object.batch_size,
)
# Evaluate performance on test data and report score
test_metrics = classifier_model.evaluate(
test_images,
test_labels,
batch_size=standard_object.batch_size,
)
test_results_dict = {out: test_metrics[i] for i, out in enumerate(classifier_model.metrics_names)}
return(test_results_dict['categorical_accuracy'])
