def train_eval_network(dataset_name, train_gen, validate_gen, test_x, test_y, seq_len, epochs, batch_size,
batch_epoch_ratio, initial_weights, size, cnn_arch, learning_rate,
optimizer, cnn_train_type, pre_weights, lstm_conf, len_train, len_valid, dropout, classes,
patience_es=15, patience_lr=5):
"""the function build, compine fit and evaluate a certain architechtures on a dataset"""
set_random_seed(2)
seed(1)
print("Experiment Runnning with CNN:",str(cnn_arch))
result = dict(dataset=dataset_name, cnn_train=cnn_train_type,
cnn=cnn_arch.__name__, lstm=lstm_conf[0].__name__, epochs=epochs,
learning_rate=learning_rate, batch_size=batch_size, dropout=dropout,
optimizer=optimizer[0].__name__, initial_weights=initial_weights, seq_len=seq_len)
print("run experimnt " + str(result))
model = BuildModel_basic.build(size=size, seq_len=seq_len, learning_rate=learning_rate,
optimizer_class=optimizer, initial_weights=initial_weights,
cnn_class=cnn_arch, pre_weights=pre_weights, lstm_conf=lstm_conf,
cnn_train_type=cnn_train_type, dropout=dropout, classes=classes)
# the network is trained on data generatores and apply the callacks when the validation loss is not improving:
# 1. early stop to training after n iteration
# 2. reducing the learning rate after k iteration where k< n
test_history = TestCallback((test_x, test_y))
history = model.fit_generator(
steps_per_epoch=int(float(len_train) / float(batch_size * batch_epoch_ratio)),
generator=train_gen,
epochs=epochs,
validation_data=validate_gen,
validation_steps=int(float(len_valid) / float(batch_size)),
callbacks=[EarlyStopping(monitor='val_loss', min_delta=0.001, patience=patience_es, ),
ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=patience_lr, min_lr=1e-8, verbose=1),
test_history
]
)
history_to_save = history.history
history_to_save['test accuracy'] = test_history.test_acc
history_to_save['test loss'] = test_history.test_loss
model_name = ""
for k, v in result.items():
model_name = model_name + "_" + str(k) + "-" + str(v).replace(".", "d")
model_path = os.path.join(res_path, model_name)
pd.DataFrame(history_to_save).to_csv(model_path + "_train_results.csv")
result['validation loss'] = min(history.history['val_loss'])
result['validation accuracy'] = max(history.history['val_acc'])
result['last validation loss'] = history.history['val_loss'][-1]
result['last validation accuracy'] = history.history['val_acc'][-1]
result['train accuracy'] = max(history.history['acc'])
result['train loss'] = min(history.history['loss'])
result['last train accuracy'] = history.history['acc'][-1]
result['last train loss'] = history.history['loss'][-1]
result['test accuracy'] = max(test_history.test_acc)
result['test loss'] = min(test_history.test_loss)
result['last test accuracy'] = test_history.test_acc[-1]
result['last test loss'] = test_history.test_loss[-1]
result['final lr'] = history.history['lr'][-1]
result['total epochs'] = len(history.history['lr'])
return result
def evaluate(dataset_name,
train_gen,
validate_gen,
test_x,
test_y,
seq_len,
epochs,
batch_size,
batch_epoch_ratio,
initial_weights,
size,
cnn_arch,
learning_rate,
optimizer,
cnn_train_type,
pre_weights,
lstm_conf,
len_train,
len_valid,
dropout,
classes,
patience_es=15,
patience_lr=5):
"""the function build, compine fit and evaluate a certain architechtures on a dataset"""
result = dict(dataset=dataset_name,
cnn_train=cnn_train_type,
cnn=cnn_arch.__name__,
lstm=lstm_conf[0].__name__,
epochs=epochs,
learning_rate=learning_rate,
batch_size=batch_size,
dropout=dropout,
optimizer=optimizer[0].__name__,
initial_weights=initial_weights,
seq_len=seq_len)
# print("run experimnt " + str(result))
model = BuildModel_basic.build(size=size,
seq_len=seq_len,
learning_rate=learning_rate,
optimizer_class=optimizer,
initial_weights=initial_weights,
cnn_class=cnn_arch,
pre_weights=pre_weights,
lstm_conf=lstm_conf,
cnn_train_type=cnn_train_type,
dropout=dropout,
classes=classes)
res = model.evaluate_generator(validate_gen,
steps=int(
float(len_valid) / float(batch_size)),
max_queue_size=10,
workers=1,
use_multiprocessing=False,
verbose=0)
print(model.metrics_names)
print(res)
def train_eval_network(dataset_name,
train_gen,
validate_gen,
test_x,
test_y,
seq_len,
epochs,
batch_size,
batch_epoch_ratio,
initial_weights,
size,
cnn_arch,
learning_rate,
optimizer,
cnn_train_type,
pre_weights,
lstm_conf,
len_train,
len_valid,
dropout,
classes,
patience_es=15,
patience_lr=5):
"""the function build, compine fit and evaluate a certain architechtures on a dataset"""
global first_called
global result
global model
global ave_acore
if first_called:
print("loading the model")
#tensorflow.set_random_seed(2)
tensorflow.random.set_random_seed(2)
seed(1)
result = dict(dataset=dataset_name,
cnn_train=cnn_train_type,
cnn=cnn_arch.__name__,
lstm=lstm_conf[0].__name__,
epochs=epochs,
learning_rate=learning_rate,
batch_size=batch_size,
dropout=dropout,
optimizer=optimizer[0].__name__,
initial_weights=initial_weights,
seq_len=seq_len)
print("run experimnt " + str(result))
model = BuildModel_basic.build(size=size,
seq_len=seq_len,
learning_rate=learning_rate,
optimizer_class=optimizer,
initial_weights=initial_weights,
cnn_class=cnn_arch,
pre_weights=pre_weights,
lstm_conf=lstm_conf,
cnn_train_type=cnn_train_type,
dropout=dropout,
classes=classes)
# the network is trained on data generatores and apply the callacks when the validation loss is not improving:
# 1. early stop to training after n iteration
# 2. reducing the learning rate after k iteration where k< n
test_history = TestCallback((test_x, test_y))
#model_path = './model_16_.h5'
#checkpoint = ModelCheckpoint(filepath=model_path,monitor='val_loss',mode='auto' ,save_best_only='True')
#default_values = dict(epoch=10,\
# learning_rate=0.0004,\
# batch_size=16,\
# optimizer=Adam,\
# initial_weights=0,\
# cnn_class=Xception,\
# pre_weights='Xavier',\
# lstm_conf=(LSTM,dict(units = 256)),\
# cnn_train_type='static'
# )
#model.compile(optimizer=default_values["optimizer"], loss='categorical_crossentropy')
h5model_path = './model_sc_2g_transfer.h5'
model.load_weights(h5model_path)
first_called = 0
# print("predicting..")
history = model.predict_generator(
validate_gen, int(float(len_valid) / float(batch_size)))
# print(history)
ave_acore = np.sum(history) / len(history)
print(ave_acore)
res_scores.append(ave_acore)
# print(model.metrics_names)
return 1
history_to_save = history.history
history_to_save['test accuracy'] = test_history.test_acc
history_to_save['test loss'] = test_history.test_loss
model_name = ""
for k, v in result.items():
model_name = model_name + "_" + str(k) + "-" + str(v).replace(
".", "d")
model_path = os.path.join(res_path, model_name)
pd.DataFrame(history_to_save).to_csv(model_path + "_train_results.csv")
result['validation loss'] = min(history.history['val_loss'])
result['validation accuracy'] = max(history.history['val_acc'])
result['last validation loss'] = history.history['val_loss'][-1]
result['last validation accuracy'] = history.history['val_acc'][-1]
result['train accuracy'] = max(history.history['acc'])
result['train loss'] = min(history.history['loss'])
result['last train accuracy'] = history.history['acc'][-1]
result['last train loss'] = history.history['loss'][-1]
result['test accuracy'] = max(test_history.test_acc)
result['test loss'] = min(test_history.test_loss)
result['last test accuracy'] = test_history.test_acc[-1]
result['last test loss'] = test_history.test_loss[-1]
result['final lr'] = history.history['lr'][-1]
result['total epochs'] = len(history.history['lr'])
return result