def build_model(emb, seq_len):
emb_layer = Embedding(
input_dim=emb.shape[0],
output_dim=emb.shape[1],
# weights=[emb],
input_length=seq_len,
# trainable=False
)
seq = Input(shape=(seq_len, ))
seq_emb = emb_layer(seq)
seq_emb = SpatialDropout1D(rate=0.2)(seq_emb)
lstm = Bidirectional(CuDNNLSTM(200, return_sequences=True))(seq_emb)
lstm_avg_pool = GlobalAveragePooling1D()(lstm)
lstm_max_pool = GlobalMaxPooling1D()(lstm)
x = Concatenate()([lstm_avg_pool, lstm_max_pool])
x = Dropout(0.2)(Activation(activation='relu')(BatchNormalization()(
Dense(1024)(x))))
out = Dense(14, activation='softmax')(x)
model = Model(inputs=seq, outputs=out)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=Lookahead(RAdam()),
metrics=['accuracy'])
return model
def __init__(self,
model,
nbActions,
explorations_iterations,
outputFile,
ensembleFolderName,
optimizer="adamax"):
self.ensembleFolderName = ensembleFolderName
self.policy = EpsGreedyQPolicy()
self.explorations_iterations = explorations_iterations
self.nbActions = nbActions
self.model = model
#Define the memory
self.memory = SequentialMemory(limit=10000, window_length=1)
#Instantiate the agent with parameters received
self.agent = DQNAgent(model=self.model,
policy=self.policy,
nb_actions=self.nbActions,
memory=self.memory,
nb_steps_warmup=200,
target_model_update=1e-1,
enable_double_dqn=True,
enable_dueling_network=True)
#Compile the agent with the optimizer given as parameter
if optimizer == "adamax":
self.agent.compile(Adamax(), metrics=['mae'])
if optimizer == "adadelta":
self.agent.compile(Adadelta(), metrics=['mae'])
if optimizer == "sgd":
self.agent.compile(SGD(), metrics=['mae'])
if optimizer == "rmsprop":
self.agent.compile(RMSprop(), metrics=['mae'])
if optimizer == "nadam":
self.agent.compile(Nadam(), metrics=['mae'])
if optimizer == "adagrad":
self.agent.compile(Adagrad(), metrics=['mae'])
if optimizer == "adam":
self.agent.compile(Adam(), metrics=['mae'])
if optimizer == "radam":
self.agent.compile(RAdam(total_steps=5000,
warmup_proportion=0.1,
min_lr=1e-5),
metrics=['mae'])
#Save the weights of the agents in the q.weights file
#Save random weights
self.agent.save_weights("q.weights", overwrite=True)
#Load data
self.train_data = pd.read_csv('./dataset/jpm/train_data.csv')
self.validation_data = pd.read_csv('./dataset/jpm/train_data.csv')
self.test_data = pd.read_csv('./dataset/jpm/test_data.csv')
#Call the callback for training, validation and test in order to show results for each iteration
self.trainer = ValidationCallback()
self.validator = ValidationCallback()
self.tester = ValidationCallback()
self.outputFileName = outputFile
def parse_net_params(filename='configs/road_signs.yml'):
params = {}
with open(filename, 'r') as ymlfile:
cfg = yaml.safe_load(ymlfile)
learning_rate = cfg['learning_rate']
if cfg['optimizer'] == 'adam':
optimizer = optimizers.Adam(lr=learning_rate)
elif cfg['optimizer'] == 'rms_prop':
optimizer = optimizers.RMSprop(lr=learning_rate)
elif cfg['optimizer'] == 'radam':
from keras_radam import RAdam
optimizer = RAdam(learning_rate)
else:
optimizer = optimizers.SGD(lr=learning_rate)
if 'augmentations_type' in cfg:
augmentations = get_aug(cfg['augmentation_type'], cfg['input_shape'])
else:
augmentations = None
params = {k: v for k, v in cfg.items() if k not in ['optimizer']}
if 'dataset_path' in cfg:
params['loader'] = EmbeddingNetImageLoader(
cfg['dataset_path'],
input_shape=cfg['input_shape'],
min_n_obj_per_class=cfg['min_n_obj_per_class'],
select_max_n_obj_per_class=cfg['select_max_n_obj_per_class'],
max_n_obj_per_class=cfg['max_n_obj_per_class'],
augmentations=augmentations)
params['optimizer'] = optimizer
return params
def build_xlnet(args):
# Load pretrained model
model = load_trained_model_from_checkpoint(
config_path=args.config_path,
checkpoint_path=args.model_path,
batch_size=args.batch_size,
memory_len=0,
target_len=args.maxlen,
in_train_phase=False,
attention_type=ATTENTION_TYPE_BI,
)
# Build classification model
last = model.output
extract = Extract(index=-1, name='Extract')(last)
output = keras.layers.Dense(units=args.nclass,
activation='softmax',
name='Softmax')(extract)
model = keras.models.Model(inputs=model.inputs, outputs=output)
model.summary()
# Fit model
model.compile(
optimizer=RAdam(args.lr),
loss='categorical_crossentropy',
metrics=['accuracy'],
)
print(model.summary())
return model
def bert_rcnn(model):
inputs = model.inputs
bert_out = NonMasking()(model.outputs)
bert_out = SpatialDropout1D(0.2)(bert_out)
l_embedding = Lambda(lambda x: K.concatenate([K.zeros(shape=(K.shape(x)[0], 1, K.shape(x)[-1])),
x[:, :-1]], axis=1))(bert_out)
r_embedding = Lambda(lambda x: K.concatenate([K.zeros(shape=(K.shape(x)[0], 1, K.shape(x)[-1])),
x[:, 1:]], axis=1))(bert_out)
forward = LSTM(256, return_sequences=True)(l_embedding)
backward = LSTM(256, return_sequences=True, go_backwards=True)(r_embedding)
backward = Lambda(lambda x: K.reverse(x, axes=1))(backward)
together = [forward, bert_out , backward]
together = Concatenate(axis=2)(together)
semantic = Conv1D(256, kernel_size=1, activation="relu")(together)
sentence_embed = Lambda(lambda x: K.max(x, axis=1))(semantic)
dense_layer = Dense(256, activation='relu')(sentence_embed)
preds = Dense(1, activation='sigmoid')(dense_layer)
model_rcnn = Model(inputs, preds)
model_rcnn.compile(loss='binary_crossentropy', optimizer=RAdam(learning_rate=2e-5), metrics=['acc'])
model_rcnn.load_weights(resource_path('256_checkpoint_rcnn2_foody.h5'))
return model_rcnn
def build_model(max_seq_length):
in_id = tf.keras.layers.Input(shape=(max_seq_length, ), name="input_ids")
in_mask = tf.keras.layers.Input(shape=(max_seq_length, ),
name="input_masks")
in_segment = tf.keras.layers.Input(shape=(max_seq_length, ),
name="segment_ids")
bert_inputs = [in_id, in_mask, in_segment]
dropout_rate = 0.5
#Freeze BERT layers for the first stage of training
bert_output = BertLayer(n_fine_tune_layers=2, trainable=False)(bert_inputs)
#Add dropout to prevent overfitting
bert_dropout = tf.keras.layers.Dropout(rate=dropout_rate)(bert_output)
dense = tf.keras.layers.Dense(256, activation="sigmoid")(bert_dropout)
dense_dropout = tf.keras.layers.Dropout(rate=dropout_rate)(dense)
pred = tf.keras.layers.Dense(classes, activation="sigmoid")(dense_dropout)
#Two extra FC layers added at the end
model = tf.keras.models.Model(inputs=bert_inputs, outputs=pred)
model.compile(loss="binary_crossentropy",
optimizer=RAdam(),
metrics=["accuracy"])
model.summary()
return model
def test_fit_embed(self):
for amsgrad in [False, True]:
model = keras.models.Sequential()
model.add(
keras.layers.Embedding(
input_shape=(None, ),
input_dim=5,
output_dim=16,
mask_zero=True,
))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=8)))
model.add(keras.layers.Dense(units=2, activation='softmax'))
model.compile(RAdam(
total_steps=38400,
warmup_proportion=0.1,
min_lr=1e-6,
weight_decay=1e-4,
amsgrad=amsgrad,
),
loss='sparse_categorical_crossentropy')
x = np.random.randint(0, 5, (1024, 15))
y = (x[:, 1] > 2).astype('int32')
model.fit(x, y, epochs=10)
model_path = os.path.join(
tempfile.gettempdir(),
'test_accumulation_%f.h5' % np.random.random())
model.save(model_path)
keras.models.load_model(model_path,
custom_objects={'RAdam': RAdam})
def create_model(self):
embedding_matrix = self.get_embeddings()
# Input
input_x = Input(shape=(self.input_size, ), dtype='int32')
# Embedding Layer
x = Embedding(len(embedding_matrix),
self.dim,
weights=[embedding_matrix],
input_length=self.input_size,
trainable=True)(input_x)
x = SpatialDropout1D(self.dropout)(x)
x = Conv1D(self.filters,
self.kernel_size,
padding='same',
activation='relu')(x)
x = GlobalMaxPool1D()(x)
x = BatchNormalization()(x)
x = Dense(750, activation='relu')(x)
out = Dense(self.output_size, activation='softmax')(x)
model = Model([input_x], out)
model.compile(loss='categorical_crossentropy',
optimizer=RAdam(lr=self.learning_rate),
metrics=['accuracy', f1_m])
model.summary()
return model
def train():
parser = argparse.ArgumentParser()
parser.add_argument('model_dir', default='model dir')
args = parser.parse_args()
model_dir = args.model_dir
hdf_dir = os.path.join(model_dir, "hdf5")
os.makedirs(hdf_dir, exist_ok=True)
bert_model_path = os.path.join(ROOT_DIR, 'BERT-baseline')
data_path = os.path.join(model_dir, "feature.pkl")
with open(data_path, 'rb') as fr:
train_data, train_label, test_data, test_label = pickle.load(fr)
print("load {}/{} train/dev items ".format(len(train_data),
len(test_data)))
bert_embed = BERTEmbedding(bert_model_path,
task=kashgari.LABELING,
sequence_length=50)
model = KashModel(bert_embed)
model.build_model(x_train=train_data,
y_train=train_label,
x_validate=test_data,
y_validate=test_label)
from src.get_model_path import get_model_path
model_path, init_epoch = get_model_path(hdf_dir)
if init_epoch > 0:
print("load epoch from {}".format(model_path))
model.tf_model.load_weights(model_path)
optimizer = RAdam(learning_rate=0.0001)
model.compile_model(optimizer=optimizer)
hdf5_path = os.path.join(hdf_dir,
"crf-{epoch:03d}-{val_accuracy:.3f}.hdf5")
checkpoint = ModelCheckpoint(hdf5_path,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
tensorboard = TensorBoard(log_dir=os.path.join(model_dir, "log"))
eval_callback = EvalCallBack(kash_model=model,
valid_x=test_data,
valid_y=test_label,
step=1,
log_path=os.path.join(model_dir, "acc.txt"))
callbacks = [checkpoint, tensorboard, eval_callback]
model.fit(train_data,
train_label,
x_validate=test_data,
y_validate=test_label,
epochs=100,
batch_size=256,
callbacks=callbacks)
return
def _set_optimizer(self, optimizer_name, lr=0.001):
if optimizer_name == 'RAdam':
opt = RAdam(learning_rate=lr)
elif optimizer_name == 'RMSprop':
opt = RMSprop(lr=lr)
elif optimizer_name == "Adam":
opt = Adam(lr=lr)
return opt
def create_step_model(world_size: int,
n_actions: int,
learning_rate: int,
neurons_fully: int,
drop_rate: float,
dueling=bool):
x = Input(shape=(3, world_size, world_size))
actions_input = Input((n_actions, ), name='mask')
x2 = (Conv2D(20,
kernel_size=3,
data_format='channels_first',
padding='same',
activation='relu',
strides=(1, 1)))(x)
x3 = (Conv2D(20,
kernel_size=3,
data_format='channels_first',
padding='valid',
activation='relu',
strides=(1, 1)))(x2)
x4 = (Conv2D(20,
kernel_size=3,
data_format='channels_first',
padding='valid',
activation='relu',
strides=(1, 1)))(x3)
out = Flatten()(x4)
if dueling == True:
F1 = (Dense(10, activation='relu'))(out)
F2 = (Dense(50, activation='relu'))(out)
y4 = (Dense(1))(F1)
y5 = (Dense(n_actions))(F2)
y8 = keras.layers.Reshape((n_actions, 1))(y5)
y9 = keras.layers.AveragePooling1D(pool_size=n_actions)(y8)
output = keras.layers.Add()([y5, y4])
output1 = keras.layers.Subtract()([output, y9])
else:
output1 = (Dense(5))(out)
actions_input2 = keras.layers.Reshape((1, n_actions))(actions_input)
filtered_output = keras.layers.Multiply()([actions_input2, output1])
filtered_output = keras.layers.Reshape((n_actions, 1))(filtered_output)
step_model = Model(inputs=[x, actions_input], outputs=filtered_output)
if learning_rate < 1:
opt = optimizers.SGD(lr=learning_rate,
decay=0,
momentum=0.9,
nesterov=True)
else:
opt = RAdam()
step_model.compile(optimizer=opt, loss='mean_squared_error')
return step_model
def create_model(self):
embedding_matrix = self.get_embeddings()
# Input
input_x = Input(shape=(self.input_size, ), dtype='int32')
# Embedding Layer
x = Embedding(len(embedding_matrix),
self.dim,
weights=[embedding_matrix],
input_length=self.input_size,
trainable=True)(input_x)
x = SpatialDropout1D(0.3)(x)
def inseption_mod(input_layer, layers=10):
### 1st layer
layer_1 = Conv1D(layers, 1, padding='same',
activation='relu')(input_layer)
layer_1 = Conv1D(layers, 3, padding='same',
activation='relu')(layer_1)
layer_2 = Conv1D(layers, 1, padding='same',
activation='relu')(input_layer)
layer_2 = Conv1D(layers, 5, padding='same',
activation='relu')(layer_2)
layer_3 = MaxPooling1D(3, strides=1, padding='same')(input_layer)
layer_3 = Conv1D(layers, 1, padding='same',
activation='relu')(layer_3)
x_rnn = LSTM(30, return_sequences=True)(input_layer)
x = concatenate([layer_1, layer_2, layer_3, x_rnn], axis=-1)
return x
x = inseption_mod(x, 30)
x = Dropout(0.3)(x)
# RNN Layer
#model.add(LSTM(seq_size))
x = GlobalMaxPool1D()(x)
x = BatchNormalization()(x)
# Dense Layer
x = Dense(256, activation='relu')(x)
x = Dropout(0.3)(x)
output = Dense(self.output_size, activation='softmax')(x)
model = Model([input_x], output)
model.compile(loss='categorical_crossentropy',
optimizer=RAdam(),
metrics=['accuracy', f1_m])
model.summary()
return model
def model_v1():
model = Sequential([
Dense(9, activation='relu', input_shape=(len(df.keys()), )),
Dense(1, activation='sigmoid'),
])
model.compile(loss=binary_crossentropy,
optimizer=RAdam(),
metrics=['mse', 'binary_accuracy'])
return model
def test_lr(self):
opt = Lookahead(RAdam())
K.set_value(opt.lr, 1e-4)
self.assertAlmostEqual(1e-4, K.get_value(opt.lr))
self.assertAlmostEqual(1e-4, K.get_value(opt.optimizer.lr))
if not TF_KERAS:
opt.lr = K.variable(1e-3)
self.assertAlmostEqual(1e-3, K.get_value(opt.lr))
self.assertAlmostEqual(1e-3, K.get_value(opt.optimizer.lr))
def gen_keras_linear(w, b, weight_decay=0.):
model = keras.models.Sequential()
model.add(keras.layers.Dense(input_shape=(3, ), units=5, name='Dense'))
model.get_layer('Dense').set_weights([w, b])
model.compile(optimizer=RAdam(
lr=1e-3,
weight_decay=weight_decay,
),
loss='mse')
return model
def get_optimizer(self):
if self.optimizer_type == 'adam':
opt = Adam(lr=self.learning_rate)
elif self.optimizer_type == 'adamax':
opt = Adamax(lr=self.learning_rate)
elif self.optimizer_type == 'radam':
opt = RAdam(lr=0.01, total_steps=5000, warmup_proportion=0.1, min_lr=0.0001)
else:
raise ValueError('Undefined OPTIMIZER_TYPE!')
return opt
def gen_linear_model() -> keras.models.Model:
model = keras.models.Sequential()
model.add(keras.layers.Dense(
input_shape=(17,),
units=3,
bias_constraint=keras.constraints.max_norm(),
name='Dense',
))
model.compile(RAdam(decay=1e-4, weight_decay=1e-4), loss='mse')
return model
def get_bert_finetuning_model(model):
inputs = model.inputs[:2] #segment, token 두개의 input
dense = model.output
out = keras.layers.Dropout(0.2)(dense)
output_layer = keras.layers.Dense(7, activation='softmax')(out)
model = keras.models.Model(inputs, output_layer)
model.compile(optimizer=RAdam(learning_rate=LR, weight_decay=0.001),
loss="categorical_crossentropy",
metrics=["categorical_accuracy"])
return model
def CNNLSTMAttention(network_input_shape, number_of_classes, optimizer, rnn_func = LSTM):
inputs = Input(shape=(int(network_input_shape[0]), int(network_input_shape[1]), 1))
x = Permute((2,1,3)) (inputs)
x = Conv2D(10, (5,1) , activation='relu', padding='same') (x)
x = BatchNormalization() (x)
x = Conv2D(1, (5,1) , activation='relu', padding='same') (x)
x = BatchNormalization() (x)
x = Lambda(lambda q: K.squeeze(q, -1), name='squeeze_last_dim') (x) #keras.backend.squeeze(x, axis)
x = Bidirectional(rnn_func(64, return_sequences = True, activation='tanh',recurrent_activation='sigmoid')) (x) # [b_s, seq_len, vec_dim]
x = Bidirectional(rnn_func(64, return_sequences = True, activation='tanh',recurrent_activation='sigmoid')) (x) # [b_s, seq_len, vec_dim]
xFirst = Lambda(lambda q: q[:,49]) (x) #[b_s, vec_dim] #32
query = Dense(128) (xFirst)
#dot product attention
attScores = Dot(axes=[1,2])([query, x])
attScores = Softmax(name='attSoftmax')(attScores) #[b_s, seq_len]
#rescale sequence
attVector = Dot(axes=[1,1])([attScores, x]) #[b_s, vec_dim]
x = Dense(64, activation = 'relu')(attVector)
x = Dense(32)(x)
output = Dense(number_of_classes, activation = 'softmax', name='output')(x)
model = Model(inputs=[inputs], outputs=[output])
# Optimizer choice
if optimizer=='radam':
Optimizer=RAdam()
elif optimizer=='sgd':
Optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False)
elif optimizer=='rmsprop':
Optimizer=keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
elif optimizer=='adagrad':
Optimizer=keras.optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
elif optimizer=='adadelta':
Optimizer=keras.optimizers.Adadelta(lr=1.0, rho=0.95,epsilon=None, decay=0.0)
elif optimizer=='adam':
Optimizer=keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
elif optimizer=='adamax':
Optimizer=keras.optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)
else:
Optimizer=keras.optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)
model.compile(optimizer=Optimizer,loss='categorical_crossentropy',metrics=['accuracy'])
return model
def __init__(self, chan_len: int, n_hidden: int, learning_rate: float):
input = Input(shape=(2 * chan_len, )) # 13*2=26個
hidden1 = Dense(n_hidden, activation='relu')(input)
output1 = Dense(1, activation='linear')(hidden1)
output2 = Dense(1, activation='linear')(hidden1)
model = Model(inputs=input, outputs=[output1, output2])
# optimizer = optimizers.Adam(lr=learning_rate)
optimizer = RAdam()
# optimizer = optimizers.SGD(learning_rate=0.001, momentum=0.8)
model.compile(optimizer, loss="mse")
self.model = model
def train(m, output_width, output_height):
#m.compile(RAdam(lr=1e-3),
# loss=dice_coef_multilabel,
# metrics=[tversky_loss, dice_coef, dice_coef_multilabel, focal_loss, categorical_focal_loss])
m.compile(
RAdam(lr=1e-4),
loss=dice_loss,
metrics=[tversky_loss, dice_coef, dice_coef_multilabel, focal_loss])
# train_gen = LoadBatches.imageSegmentationGenerator(train_images_path, train_segs_path,
# train_batch_size, n_classes,
# input_height, input_width,
# output_height, output_width)
train_gen = LoadBatches.augmentedDataGenerator(train_images_path,
train_segs_path,
train_batch_size, n_classes,
input_height, input_width,
output_height, output_width)
val_gen = LoadBatches.imageSegmentationGenerator(
val_images_path, val_segs_path, val_batch_size, n_classes,
input_height, input_width, output_height, output_width)
print()
model_checkpoint = ModelCheckpoint(checkpoint_filepath,
monitor='val_loss',
verbose=1,
save_best_only=True)
model_tensorboard = TensorBoard(log_dir='./Tensorboard/logs',
histogram_freq=0,
batch_size=train_batch_size,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None,
embeddings_data=None,
update_freq='batch')
gbytes = get_model_memory_usage(train_batch_size, m)
print('Total estimated memory GBtytes: %f' % gbytes)
m.fit_generator(train_gen,
steps_per_epoch=int(78 / train_batch_size),
validation_data=val_gen,
callbacks=[model_checkpoint, model_tensorboard],
validation_steps=int(20 / val_batch_size),
epochs=epochs)
m.save_weights(save_weights_path + "finalweights.hdf5")
m.save(save_weights_path + ".model.finalweights.hdf5")
def callRectifiedAdam(learnRate, beta1, beta2, epsilon, decay, weightDecay,
amsgrad, totalSteps, warmUpProp, minLr):
return RAdam(lr=learnRate,
beta_1=beta1,
beta_2=beta2,
epsilon=epsilon,
decay=decay,
weight_decay=weightDecay,
amsgrad=amsgrad,
total_steps=totalSteps,
warmup_proportion=warmUpProp,
min_lr=minLr)
def get_bert_finetuning_model(model):
inputs = model.inputs[:2]
dense = model.output
x = NonMasking()(dense)
outputs_start = MyLayer_Start(SEQ_LEN)(x)
outputs_end = MyLayer_End(SEQ_LEN)(x)
bert_model = keras.models.Model(inputs, [outputs_start, outputs_end])
bert_model.compile(optimizer=RAdam(learning_rate=LR, decay=0.0001),
loss='categorical_crossentropy',
metrics=['accuracy'])
return bert_model
def LSTM_CNN_RAdam(self):
self.model_name = "LSTM_CNN_RAdam"
model = Sequential()
model.add(layers.Embedding(self.max_features, self.output_embedding))
model.add(layers.Dropout(self.dropout))
model.add(layers.Conv1D(2**10, 4, padding='valid', activation='relu', strides=1))
model.add(layers.MaxPooling1D(pool_size=4))
model.add(layers.LSTM(self.output_embedding))
model.add(layers.Dense(self.output_neuron))
model.add(layers.Activation('sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy', optimizer=RAdam(), metrics=['accuracy'])
self.model = model
def get_bert_finetuning_model(model):
inputs = model.inputs[:2]
bert_transformer = model.layers[-1].output
x = NonMasking()(bert_transformer)
outputs_start, outputs_end = answer_predict(512)(x)
bert_qa_model = keras.models.Model(inputs, [outputs_start, outputs_end])
optimizer_warmup = RAdam(learning_rate=1e-5,
warmup_proportion=0.1,
epsilon=1e-6,
weight_decay=0.01)
bert_qa_model.compile(optimizer=optimizer_warmup,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return bert_qa_model
def test_ranger(self):
model = self._init_model(Lookahead(RAdam()))
x, y, w = self._init_data(data_size=100000)
model.fit(x, y, epochs=5)
model_path = os.path.join(tempfile.gettempdir(), 'test_lookahead_%f.h5' % np.random.random())
model.save(model_path)
model: keras.models.Model = keras.models.load_model(model_path, custom_objects={
'RAdam': RAdam,
'Lookahead': Lookahead,
})
x, y, _ = self._init_data(data_size=100, w=w)
predicted = model.predict(x)
self.assertLess(np.max(np.abs(predicted - y)), 1e-3)
def train_model(batch_size, num_classes, epochs, learning_rate):
'''
Tasks :
1. Load the training and test dataset from the root folder directory
2. Randomly Undersample the training data to prevent bias due to
class imbalance
3. Define the Model Architecture
4. Train the model
5. Save the model and history to get a visualisation of model
performance over the period of training
'''
# DataPath
path = './data/NPY-Files/'
X_train = np.load(path + 'X_train.npy')
y_train = np.load(path + 'Y_train.npy')
print('Loaded Train Data')
print(X_train.shape, y_train.shape)
# Loading Test Data For Validation
X_test = np.load(path + 'X_test.npy')
y_test = np.load(path + 'Y_test.npy')
print('Loaded Test Data')
print(X_test.shape, y_test.shape)
# Converting to Categorical Data
y_train = to_categorical(y_train, num_classes = num_classes)
y_test = to_categorical(y_test, num_classes = num_classes)
# Creating model
classifier = model.init_model(num_classes = num_classes)
optimizer = RAdam(lr = LEARNING_RATE)
# Compiling model with optimizer, loss and metrics
classifier.compile(loss = focal_loss, optimizer = optimizer, metrics = ["accuracy"])
callbacks = [
keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, write_graph = True, write_images = True),
keras.callbacks.ModelCheckpoint(os.path.join(WEIGHT_SAVE_PATH, 'weights{epoch:08d}.h5'),
verbose=0, save_weights_only=True),
keras.callbacks.EarlyStopping(monitor='val_loss', patience = 5, verbose = 1)]
# Starting training of model on undersampled training data
history = classifier.fit(X_train, y_train, batch_size=batch_size,
epochs=epochs, verbose=1, validation_data = (X_test, y_test), callbacks = callbacks, initial_epoch = initial_epochs)
def get_bert_finetuning_model(model):
inputs = model.inputs[:2]
dense = model.layers[-3].output
outputs = keras.layers.Dense(
1,
activation='sigmoid',
kernel_initializer=keras.initializers.TruncatedNormal(stddev=0.02),
name='real_output')(dense)
bert_model = keras.models.Model(inputs, outputs)
bert_model.compile(optimizer=RAdam(learning_rate=0.00001,
weight_decay=0.0025),
loss='binary_crossentropy',
metrics=['accuracy'])
return bert_model
def build_model():
"""
A custom implementation of EfficientNetB5
for the APTOS 2019 competition
(Regression)
"""
model = Sequential()
model.add(effnet)
model.add(GlobalAveragePooling2D())
model.add(Dropout(0.5))
model.add(Dense(5, activation=elu))
model.add(Dense(1, activation="linear"))
model.compile(loss='mse',
optimizer=RAdam(learning_rate=0.00005),
metrics=['mse', 'acc'])
print(model.summary())
return model
def test(m, output_width, output_height):
K.set_learning_phase(0)
m.compile(RAdam(lr=1e-4), loss=dice_loss, metrics=[dice_coef])
images = glob.glob(test_images_path + "*.jpg") + glob.glob(
test_images_path + "*.png") + glob.glob(test_images_path + "*.jpeg")
images.sort()
segmentations = glob.glob(test_segs_path + "*.jpg") + glob.glob(
test_segs_path + "*.png") + glob.glob(test_segs_path + "*.jpeg")
segmentations.sort()
for i in range(len(images)):
im_path = images[i]
seg_path = segmentations[i]
data = LoadBatches.getImageArr(im_path, input_width, input_height)
data = np.expand_dims(data, axis=0)
predictions = m.predict(data)
seg = predictions[0]
#seg=np.reshape(seg,(output_width, output_height, n_classes))
threshold = 0.5
img = np.zeros((output_width, output_height, 3), int)
if (n_classes == 2):
#img[:,:,0]=(seg[:,:,1] >= threshold).astype(int)*255
#Bladder
img[:, :, 0] = (seg[:, :, 1] >= threshold).astype(int) * 1
img[:, :, 1] = (seg[:, :, 1] >= threshold).astype(int) * 1
img[:, :, 2] = (seg[:, :, 1] >= threshold).astype(int) * 1
elif (n_classes == 4):
#img[:,:,0]=seg[:,:,1]*255
#img[:,:,1]=seg[:,:,2]*255
#img[:,:,2]=seg[:,:,3]*255
img[:, :, 0] = (seg[:, :, 1] >= threshold).astype(int) * 255
img[:, :, 1] = (seg[:, :, 2] >= threshold).astype(int) * 255
img[:, :, 2] = (seg[:, :, 3] >= threshold).astype(int) * 255
else:
for c in range(n_classes):
img[:, :, 0] = (seg[:, :, c] >= threshold).astype(int) * c
img[:, :, 1] = (seg[:, :, c] >= threshold).astype(int) * c
img[:, :, 2] = (seg[:, :, c] >= threshold).astype(int) * c
filename = seg_path
cv2.imwrite(filename, img)
K.clear_session()