def test_epochs(e=[10, 12, 14, 16, 18, 20, 22, 24, 26]):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
results = {}
results_validation = {}
for i in range(len(e)):
temp_results = []
temp_results2 = []
for j in range(20):
model = build_model(layers_neurons=[32, 4, 1],
lr=0.001,
momentum=0.99)
history = model.fit(X_train, y_train, epochs=e[i], batch_size=10)
temp_results.append(history.history["loss"][-1])
temp_results2.append(model.evaluate(X_valid, y_valid))
results[e[i]] = mean(temp_results)
results_validation[e[i]] = mean(temp_results2)
with open('results.json', "r") as file:
data = json.load(file)
data["epochs"] = results
data["epochs_valid"] = results_validation
with open('results.json', "w") as file:
json.dump(data, file, indent=4)
return results
def test_params(param_to_test,
save_file="results",
sample_size=20,
fln=[32],
sln=[4],
lr=[0.001],
mom=[0.99],
e=[20],
bs=[20]):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
results = []
for i in range(len(fln)):
for j in range(len(sln)):
for k in range(len(lr)):
for l in range(len(mom)):
for m in range(len(e)):
for n in range(len(bs)):
train_mean, train_std, valid_mean, valid_std = test(
X_train,
y_train,
X_valid,
y_valid,
sample=sample_size,
fln=fln[i],
sln=sln[j],
lr=lr[k],
m=mom[l],
e=e[m],
bs=bs[n])
result = {
"first_layer_neurons": fln[i],
"second_layer_neurons": sln[j],
"learning_rate": lr[k],
"momentum": mom[l],
"epochs": e[m],
"batch_size": bs[n],
"sample_size": sample_size,
"train_mean": train_mean,
"train_std": train_std,
"valid_mean": valid_mean,
"valid_std": valid_std
}
results.append(result)
file_name = save_file + datetime.now().strftime(
"%d_%m_%Y_%H_%M_%S") + ".json"
data = {param_to_test: results}
with open(file_name, "w") as file:
json.dump(data, file, indent=4)
return results
def evaluate_model_on_noisy_data(train_models=False, plot_history=False):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
X_train_noise, X_valid_noise, X_test_noise, y_train_noise, y_valid_noise, y_test_noise = get_datasets(
with_noise=True)
dt = datetime.now().strftime("%d_%m_%Y_%H_%M_%S")
if train_models:
model, history = train_model(X_train, y_train, "model{}.h5".format(dt),
"trainHistoryDict{}".format(dt))
noisy_model, noisy_history = train_model(
X_train_noise, y_train_noise, "noisy_model{}.h5".format(dt),
"noisy_trainHistoryDict{}".format(dt))
else:
model = load_model("model27_01_2020_16_45_06.h5")
noisy_model = load_model("noisy_model27_01_2020_16_45_06.h5")
with open("trainHistoryDict27_01_2020_16_45_06", "rb") as file:
history = pickle.load(file)
with open("noisy_trainHistoryDict27_01_2020_16_45_06", "rb") as file:
noisy_history = pickle.load(file)
if plot_history:
plot_loss_history(history, "loss_during_training{}".format(dt))
plot_loss_history(noisy_history,
"loss_during_training_noise{}".format(dt),
with_noise=True)
metrics = []
metrics.append(["Train"] +
calculate_metrics(model.predict(X_train), y_train))
metrics.append(["Validation"] +
calculate_metrics(model.predict(X_valid), y_valid))
metrics.append(["Test"] + calculate_metrics(model.predict(X_test), y_test))
metrics.append(
["Train with noise"] +
calculate_metrics(noisy_model.predict(X_train_noise), y_train_noise))
metrics.append(
["Validation with noise"] +
calculate_metrics(noisy_model.predict(X_valid_noise), y_valid_noise))
metrics.append(
["Test with noise"] +
calculate_metrics(model.predict(X_test_noise), y_test_noise))
save_metrics(dt, metrics)
def test_learning_rate(lrs=[0.1, 0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001]):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
results = {}
for i in range(len(lrs)):
temp_results = []
for j in range(20):
model = build_model(layers_neurons=[32, 4, 1], lr=lrs[i])
history = model.fit(X_train, y_train, epochs=20, batch_size=10)
temp_results.append(history.history["loss"][-1])
results[lrs[i]] = mean(temp_results)
with open('results.json', "r") as file:
data = json.load(file)
data["learning_rate"] = results
with open('results.json', "w") as file:
json.dump(data, file, indent=4)
return results
def test_second_layer_size(max_size=6):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
results = {}
for i in range(max_size):
temp_results = []
for j in range(20):
model = build_model(layers_neurons=[2**(max_size - 1), 2**i, 1])
history = model.fit(X_train, y_train, epochs=20, batch_size=10)
temp_results.append(history.history["loss"][-1])
results[2**i] = mean(temp_results)
with open('results.json', "r") as file:
data = json.load(file)
data["second_layer_size"] = results
with open('results.json', "w") as file:
json.dump(data, file, indent=4)
return results
def run(train_new_model=False,
model_file='model.h5',
history_file='trainHistoryDict',
evaluation='manual'):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
if evaluation == 'keras':
evaluate_with_keras_pipeline(X_valid, y_valid)
else:
if train_new_model:
model, history = train_model(X_train, y_train, model_file,
history_file)
else:
model = load_model(model_file)
with open(history_file, "rb") as file:
history = pickle.load(file)
evaluate_manually(X_train, y_train, X_valid, y_valid, model, history)
return model, history
def test_momentum(m=[0.99, 0.97, 0.95, 0.93, 0.91]):
X_train, X_valid, X_test, y_train, y_valid, y_test = get_datasets()
results = {}
for i in range(len(m)):
temp_results = []
for j in range(20):
model = build_model(layers_neurons=[32, 4, 1],
lr=0.001,
momentum=m[i])
history = model.fit(X_train, y_train, epochs=20, batch_size=10)
temp_results.append(history.history["loss"][-1])
results[m[i]] = mean(temp_results)
with open('results.json', "r") as file:
data = json.load(file)
data["momentum"] = results
with open('results.json', "w") as file:
json.dump(data, file, indent=4)
return results