def prep_student(dataset, verbose, alpha, temperature):
# if student is not saved beforehand, train and save it
model_path = '{}/models/student/'.format(dataset)
if not isfile(model_path +
'model/model.h5') or not isfile(model_path +
'model/model.json'):
print('Student model doesnt exist, training it...')
# load dataset and logits
_dataset = get_data(dataset)
x_train, y_train, x_test, y_test = _dataset.get_data()
train_features = np.load(
'{}/models/teacher/npy/train_logits.npy'.format(dataset))
test_features = np.load(
'{}/models/teacher/npy/test_logits.npy'.format(dataset))
# normalized output with temperature shape=(num_samples,num_classes)
y_train_soft = softmax(train_features / temperature)
y_test_soft = softmax(test_features / temperature)
# concatenated output labels=(num_samples,2*num_classes)
y_train_new = np.concatenate([y_train, y_train_soft], axis=1)
y_test_new = np.concatenate([y_test, y_test_soft], axis=1)
# build student model
student = get_model(_dataset, 'distillation', is_dropout=True)
# remove softmax
student.layers.pop()
# get features
logits = student.layers[-1].output
# normal softmax output
probs = Activation('softmax')(logits)
# softmax output with temperature
logits_T = Lambda(lambda x: x / temperature)(logits)
probs_T = Activation('softmax')(logits_T)
# concatanete
output = concatenate([probs, probs_T])
# This is our new student model
student = Model(student.input, output)
compile_model(student,
loss=distillation_loss(_dataset.num_classes, alpha),
metrics=[acc_distillation(_dataset.num_classes)])
# create a new dataset with generated data
dataset_s = DatasetCls(x_train,
y_train_new,
x_test,
y_test_new,
dataset_name=dataset)
# train student
student = train(dataset_s,
student,
PARAMS[dataset]['epochs'] * 2,
PARAMS[dataset]['batch_size'],
log_dir=model_path,
callbacks=[
early_stop(patience=PARAMS[dataset]['patience'],
monitor='val_loss',
verbose=verbose)
],
verbose=verbose)
# save output files
save_model_outputs(student, _dataset, model_path)
K.clear_session()
def classification_model(self,
num_dense_layers=2,
num_dense_units=256,
dropout_rate=0.2,
num_classes=7,
pooling='avg',
name='default_resnet_classification_model',
input_shape = 256,
save_to = "",
lr=1e-4
**kwargs):
""" Returns a classifier model using the correct backbone.
"""
if isinstance(input_shape, int):
if K.image_data_format() == 'channels_last':
input_shape = (input_shape, input_shape, 3)
else:
input_shape = (3, input_shape, input_shape)
else :
t = keras.layers.Input(shape = input_shape)
#input shape is processed in base_model
base_model = self.build_base_model(input_tensor=t)
x = base_model.ouput
#output is softmax output of last dense layer
output = default_classification_model(input_tensor=x,
input_shape=base_model.ouput_shape[1:],
num_classes=num_classes,
num_dense_layers=num_dense_layers,
num_dense_units=num_dense_units,
dropout_rate=dropout_rate,
pooling=pooling,
kernel_regularizer='L2'
)
model = keras.models.Model(inputs = t, outputs=output, name=name)
if save_to:
save_model_to_run(model, save_to)
compile_model(model=model,
num_classes=num_classes,
metrics='acc',
loss='ce',
lr=lr)
return model
def validate(self):
""" Checks whether the backbone string is correct.
"""
allowed_backbones = ['resnet50', ]
if self.backbone_name not in allowed_backbones:
raise ValueError('Backbone (\'{}\') not in allowed backbones ({}).'.format(self.backbone_name,
allowed_backbones))
def classification_model(self,
num_dense_layers=2,
num_dense_units=256,
dropout_rate=0.2,
num_classes=7,
pooling='avg',
name='default_resnet_classification_model',
input_shape = 256,
save_to = "",
lr=1e-4,
kernel_regularizer=None
):
""" Returns a classifier model using the correct backbone.
"""
if isinstance(input_shape, int):
if K.image_data_format() == 'channels_last':
input_shape = (input_shape, input_shape, 3)
else:
input_shape = (3, input_shape, input_shape)
t = keras.layers.Input(shape = input_shape)
#input shape is processed in base_model
base_model = self.build_base_model(input_tensor=t)
x = base_model.ouput
#output is softmax output of last dense layer
output = default_classification_model(input_tensor=x,
input_shape=base_model.ouput_shape[1:],
num_classes=num_classes,
num_dense_layers=num_dense_layers,
num_dense_units=num_dense_units,
dropout_rate=dropout_rate,
pooling=pooling,
kernel_regularizer=kernel_
)
model = keras.models.Model(inputs = t, outputs=output, name=name)
if save_to:
save_model_to_run(model, save_to)
compile_model(model=model,
num_classes=num_classes,
metrics='acc',
loss='ce',
lr=lr)
return model
def get_model(self) -> Model:
"""Get selected model.
Returns
-------
Model
Keras model object, compiled.
"""
# Build the model
optimizer = RMSprop(self.learning_rate)
if self.model_type == "unet":
model = models.unet(
self.input_shape,
self.classes,
self.loss,
)
elif self.model_type == "unet_large":
model = models.unet_large(
self.input_shape,
self.classes,
self.loss,
)
elif self.model_type == "fcdensenet":
model = models.fcdensenet(
self.input_shape,
self.classes,
self.loss,
)
elif self.model_type == "fcn_small":
model = models.fcn_small(self.input_shape, self.classes, self.loss)
if self.preload_weights:
logger.info(
"=====================================================")
logger.info(f"Using weights from {self.preload_weights}")
logger.info(
"=====================================================")
model.load_weights(self.preload_weights)
if config.FINE_TUNE_AT > 0:
for layer in model.layers[:config.FINE_TUNE_AT]:
layer.trainable = False
logger.info(
"=====================================================")
logger.info(f"Fine tuning from %d layer" % config.FINE_TUNE_AT)
logger.info(
"=====================================================")
model = models.compile_model(model, self.loss, optimizer)
model.run_eagerly = True
model.summary()
return model
def run(self):
model_id = self.job['model']
categories = self.job['categories']
filename, architecture = get_model(model_id)
cat_dict, categories_id = request_categories(categories)
path = prepare_dataset(categories_id, filename)
model, gen = compile_model(architecture, categories_id)
train_generator, test_generator, image_files, class_indices = create_gens(
path, gen)
train_accuracy, train_loss, val_accuracy, val_loss = start_training(
model, train_generator, test_generator, image_files, filename)
create_labels(cat_dict, filename, class_indices)
api.update_job(
self.job['id'], {
'done': 1,
'train_accuracy': train_accuracy,
'train_loss': train_loss,
'val_accuracy': val_accuracy,
'val_loss': val_loss
})
def main(
train_x,
train_y,
patch_rows,
patch_cols,
batch_size=64,
epochs=1000,
model="h_unet",
name_suffix="suffix",
):
tmp_model_path = os.path.join(
'cache', name_suffix + '{epoch:02d}-{val_loss:.2f}_ckpt_best.hdf5')
early_stopping_patience = 15
model_args = {
'optimizer': 'Nadam',
'input_shape': (patch_rows, patch_cols, 13),
'base_depth': 64,
'lr': 0.0001
}
# reduce base_depth to 32 if using vanilla unet
if model == 'unet':
model_args['base_depth'] = 32
# (1064, 13, 900, 900)
print(train_x.shape)
print(train_y.shape)
# create generators for training and validation
training_gen = DataGenerator(train_x[100:],
train_y[100],
batch_size=batch_size,
img_rows=patch_rows,
img_cols=patch_cols,
horizontal_flip=True,
vertical_flip=True,
swap_axis=True)
validation_gen = DataGenerator(
train_x[0:100, :, :, :],
train_y[0:100, :, :, :],
batch_size=batch_size,
)
n_train_ims = train_x.shape[0] - 100
n_val_ims = 100
monitor = 'val_loss'
print()
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print(" BEGINNING MODEL TRAINING")
print(" MODEL ARCHITECTURE: {}".format(model))
print(" OPTIMIZER: {}".format(model_args['optimizer']))
print(" INPUT SHAPE: {}".format(model_args['input_shape']))
print(" BATCH SIZE: {}".format(batch_size))
print(" LEARNING RATE: {}".format(model_args['lr']))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print()
callbax = []
callbax.append(
ReduceLROnPlateau(factor=0.2, patience=3, verbose=1, min_delta=0.01))
# callbax.append(ModelCheckpoint(tmp_model_path, monitor=monitor))
callbax.append(TerminateOnMetricNaN('precision'))
callbax.append(
EarlyStopping(monitor=monitor,
patience=early_stopping_patience,
mode='auto'))
callbax.append(TensorBoard(log_dir='tensorboard'))
lf = jaccard_coef_loss
am = [precision, recall, jaccard_coef_int]
original_model, parallel_model = compile_model(arch=model,
loss_func=lf,
additional_metrics=am,
verbose=True,
**model_args)
callbax.append(MyCbk(original_model, tmp_model_path))
# plot_model(model, to_file='scheme.png', show_shapes=True)
# model = load_model('cache/8_1000_Atlanta_nadir44_catid_1030010003CCD70076-0.47_ckpt_best.hdf5', custom_objects={
# 'jaccard_coef_loss': jaccard_coef_loss,
# 'jaccard_coef_int': jaccard_coef_int,
# 'precision': precision,
# 'recall': recall})
parallel_model.fit_generator(training_gen,
validation_data=validation_gen,
validation_steps=np.floor(n_val_ims /
batch_size),
steps_per_epoch=400,
epochs=epochs,
workers=8,
callbacks=callbax)
original_model.save(os.path.join('cache', 'model' + name_suffix + '.hdf5'))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print(" MODEL TRAINING COMPLETE! ")
print(" Model located at {}".format(name_suffix))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
word_emb_dim)
#construct student models for different stages
with strategy.scope():
model_1 = models.construct_bilstm_student_model(
distil_tokenizer.word_index,
word_emb,
max_seq_length,
bilstm_hidden_size,
config.hidden_size,
dropout_rate,
dense_act_func,
len(label_list),
stage=1,
do_NER=do_NER)
model_1 = models.compile_model(model_1, strategy, stage=1)
model_2 = models.construct_bilstm_student_model(
distil_tokenizer.word_index,
word_emb,
max_seq_length,
bilstm_hidden_size,
config.hidden_size,
dropout_rate,
dense_act_func,
len(label_list),
stage=2,
do_NER=do_NER)
model_2 = models.compile_model(model_2, strategy, stage=2)
#get shared layers for student models
print(y_test.shape)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
n_classes = len(np.unique(y_train))
#model = models.get_model(width, height, channels, n_classes, hl=6, max_pool=False, nnodes=220)
model = models.get_vgg_model2(width,
height,
channels,
n_classes,
hl=6,
max_pool=False,
nnodes=220)
model = models.compile_model(model)
n_epochs = 50
history = model.fit(x_train, y_train, batch_size=20, epochs=n_epochs)
test_scores = model.evaluate(x_test, y_test, verbose=0)
test_scores.append(
history.history['sparse_categorical_accuracy'][n_epochs - 1])
results.append(test_scores)
tf.keras.backend.clear_session()
iteration_end_time = time.time()
iteration_time = (iteration_end_time - iteration_start_time) / 60
total_elapsed_time = (iteration_end_time - start_time) / 60
est_complete = (total_elapsed_time / (idx + 1)) * (num_iterations - idx)
print(f'Iteration Time: {iteration_time}')
print(f'Total Elapsed Time: {total_elapsed_time}')
print(f'Est. Time Remaining: {est_complete}')
import numpy as np
BATCH_SIZE = 1000
TXT_DIM = 4092
MODEL_PATH = "./models/main_{}.h5".format(TXT_DIM)
"""
change file paths here
"""
txt_train = np.load('../Data/txt_train_trunc_{}.npy'.format(TXT_DIM))
txt_val = np.load('../Data/txt_val_trunc_{}.npy'.format(TXT_DIM))
img_train = np.load('../Data/img_train.npy')
img_val = np.load('../Data/img_val.npy')
model = create_model(TXT_DIM)
print(model.count_params())
compile_model(model)
print("done")
history = model.fit(x=[txt_train,img_train], y=[np.ones((txt_train.shape[0],1024))], \
validation_data = ([txt_val, img_val],[np.ones((1000,1024))]), \
batch_size = BATCH_SIZE, epochs = 50, verbose = 2, shuffle = True)
# Plot history
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='loss (testing data)')
plt.plot(history.history['val_loss'], label='loss (validation data)')
plt.title('Training losses')
plt.ylabel('loss value')
plt.xlabel('No. epoch')
plt.legend(loc="upper left")
# Split train data into input (X) and output (Y) variables.
X_train = train[:, 1:3197]
y_train = train[:, 0]
# Split test data into input (X) and output (Y) variables.
X_test = test[:, 1:3197]
y_test = test[:, 0]
# Normalize train and test features
X_train, X_test = normalize_data(X_train, X_test)
# Create model.
model = build_model(gpus, units, dropout)
# Compile model.
model = compile_model(model, lr_rate)
# Fit model.
model = fit_model(model, loss_patience, X_train, y_train, X_test, y_test)
# Evaluate training data on the model.
validate_data("Train", X_train, y_train)
# Evaluate test data on the model.
validate_data("Test", X_test, y_test)
# Predict our test dataset.
predictions = model.predict(X_test)
# Output our test dataset for visualization.
print_predictions(predictions, print_results)
