def evaluate_model(train, test, n_input, save_model_path, save_plot_path):
# fit model
model = build_model(train, n_input)
# history is a list of yearly data
history = [x for x in train]
# walk-forward validation over period
predictions = list()
for i in range(len(test)):
# predict the period
yhat_sequence = forecast_multichannel(model, history, n_input)
# store the predictions
predictions.append(yhat_sequence)
# get real observation and add to history for predicting the next period
history.append(test[i, :])
# get array of predictions
prediction = np.array(predictions)
prediction = np.ravel(prediction)
# get array of actual values from test set
actual = test[:, :, 0]
actual_plot = np.ravel(actual)
actual[actual == 0] = np.nanmean(actual)
actual = np.ravel(actual)
# calaculate and print scores
rmse, mape = calculate_scores(actual, prediction)
print('RMSE: %.3f' % rmse)
print('MAPE: %.3f' % mape)
# save plot in /temp/ path as png file
save_plot(actual_plot, prediction, save_plot_path)
plot_prediction(actual_plot, prediction)
# save model
model.save(save_model_path)
def evaluate_model(train, test, model):
n_input = 365
# fit model
# model = build_model(train)
# history is a list of yearly data
history = [x for x in train]
# walk-forward validation over each year
prediction = list()
for i in range(len(test)):
# predict the year
yhat_sequence = forecast(model, history, n_input)
# store the predictions
prediction.append(yhat_sequence)
# get real observation and add to history for predicting the next year
history.append(test[i, :])
# get array of predictions
prediction = np.array(prediction)
prediction = np.ravel(prediction)
# get array of actual values from test set
actual = test[:, :, 0]
actual[actual == 0] = np.nanmean(actual)
actual = np.ravel(actual)
# calaculate and print scores
rmse, mape = calculate_scores(actual, prediction)
print('RMSE: %.3f' % rmse)
print('MAPE: %.3f' % mape)
# plot prediction
plot_prediction(actual, prediction)
# clear keras model
K.clear_session()
return mape
def process_video(self, loader, save_images=True):
psnr_all = []
while not loader.is_done():
input_sequence, gt, gt_path = loader.get_sequence()
gen = self.g_model(input_sequence)
psnr_frame = calculate_psnr(gen, gt, val_range=2)
psnr_all.append(psnr_frame)
if save_images:
gen_folder = makedir(join(c.IMG_SAVE_DIR, gt_path.split('/')[-2]))
gen_path = join(gen_folder, gt_path.split('/')[-1])
gen_uint8 = float32_to_uint8(gen)
imsave(gen_path, var2np(gen_uint8))
psnr_mean = np.mean(psnr_all)
# Plotting
video_name = gt_path.split('/')[-2]
plot_prediction(video_name, psnr_all)
return psnr_mean
# Load dataset, model and evaluation metric
train_data, test_data, softmax_classifier, accuracy = _initialize(DATA_NAME)
train_x, train_y = train_data
if DATA_NAME == 'Digit':
train_x, mean_img = train_x
num_data, num_features = train_x.shape
num_label = int(train_y.max()) + 1
print('# of Training data : %d \n' % num_data)
# Make model & optimizer
model = softmax_classifier(num_features, num_label)
optim = optimizer(OPTIMIZER, gamma=gamma, epsilon=epsilon)
# TRAIN
loss = model.train(train_x, train_y, num_epochs, batch_size, learning_rate,
optim)
print('Training Loss at last epoch: %.4f' % loss)
# EVALUATION
test_x, test_y = test_data
pred = model.eval(test_x)
acc = accuracy(pred, test_y)
print(OPTIMIZER, ' Accuracy on Test Data : %.2f' % acc)
# Visualize predictions for Digit dataset
if show_plot and DATA_NAME == 'Digit':
test_x = test_x[:, 1:] + mean_img
plot_prediction(test_x, pred, test_y)
loss = ff.CrossEntropyLoss()
else:
raise ValueError('Unknown loss.')
DATASET_SIZE = 1000
## Generate dataset
train_input, train_target, test_input, test_target, test_input_raw = generate_data(DATASET_SIZE, one_hot=ONE_HOT, normalize=True)
## Create model
model = Net(nb_nodes = 25)
print(model)
if args.no_train:
## Load best model
model.load('../model/best-model.pt')
model.eval() # Set model to eval mode
## Ploting results of best model
plot_prediction(test_input, test_input_raw, test_target, model)
plt.suptitle('Prediction of the best model')
plt.show()
else:
print('Using : {}Loss\n'.format(args.loss))
## Training model
model.train_(train_input, train_target, test_input, test_target, epoch=100, eta=1e-1, criterion=loss)
model.eval() # Set model to eval mode
## Ploting results of model at the end of training
plot_results(model.sumloss, model.train_error, model.test_error)
plot_prediction(test_input, test_input_raw, test_target, model)
plt.suptitle('Prediction of the trained model')
plt.show()
look_back,
look_ahead,
batch_size=1)
mse_losses = []
mae_losses = []
residuals = []
for data, target in train_loader:
mse_loss, mae_loss, resid = model.step(data,
target,
predict=True,
scaler=scaler)
mse_losses.append(mse_loss)
mae_losses.append(mae_loss)
residuals.append(resid)
mse_train = np.mean(mse_losses)
mae_train = np.mean(mae_losses)
print('Train (log): RMSE={:.3f}, MAE={:.3f}'.format(np.sqrt(mse_train),
mae_train))
print('Test: RMSE={:.3f}, MAE={:.3f}'.format(
np.sqrt(mean_squared_error(pred_scaled, target_scaled)),
mean_absolute_error(pred_scaled, target_scaled)))
# plot prediction
residuals_scaled = scaler.inverse_transform(
np.array(residuals).reshape(-1, 1)).squeeze()
#plot_prediction(target_scaled, pred_scaled, residuals_scaled) does not show the residuals correctly
plot_prediction(target_pred.detach().view(-1, 1).numpy(),
pred.detach().view(-1, 1).numpy(), residuals)
nima_net.load_state_dict(torch.load(MODEL_LOAD_PATH))
print(nima_net)
# -- evaluation --
predictions, loss = eval(
model=nima_net,
dataloaders=nima_dataloaders,
criterion=criterion,
device=DEVICE,
)
for phase in TEST_PHASE_LIST:
predictions[phase].insert(
loc=0,
column=AVADataset._label_key_image_id,
value=ava_datasets[phase].labels[AVADataset._label_key_image_id])
with open(EVAL_PRED_PATH, "wb") as fw:
pickle.dump(predictions, fw)
with open(EVAL_LOSS_PATH, "wb") as fw:
pickle.dump(loss, fw)
image = ava_datasets[TEST][0]["image"]
ground_truth = ava_datasets[TEST][0]["ground_truth"]
prediction = predictions[TEST].loc[0, AVADataset._label_key_ratings]
plot_prediction(image=image,
ground_truth=ground_truth,
prediction=prediction)
print("predicted average score: {}".format(ave_rating(prediction)))
print("ground truth score: {}".format(ave_rating(ground_truth)))
def model_processing(X_train, y_train, x_dim, batch_size, batches, epochs,
model_dir, model_name):
#build
model = eval(model_name)
model = model(x_dim)
# compile
opt = keras.optimizers.Adam(learning_rate=0.01)
model.compile(loss='mse', optimizer=opt, metrics=['mean_squared_error'])
#create folder where to save weights. It is deleted in every re-run.
weights_dir = model_dir + '/weights/'
try:
shutil.rmtree(weights_dir)
os.makedirs(weights_dir)
except:
os.makedirs(weights_dir)
#callbacks
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.5,
patience=5,
min_lr=0.001)
batches_period = 1 #save weights every 1 batch
loss_hist = LossHistory()
weight_saver = WeightsSaver(batches_period, weights_dir)
callbacks = [reduce_lr, weight_saver, loss_hist]
# fit
history_PerEpoch = model.fit(X_train,
y_train,
validation_data=(X_train, y_train),
epochs=epochs,
callbacks=callbacks,
shuffle=False,
batch_size=batch_size)
# log metrics and evaluation
history_PerEpoch = set_index_intervals(
pd.DataFrame(history_PerEpoch.history), batches, epochs)
history_PerBatch = pd.DataFrame(loss_hist.losses)
eval_PerEpoch_OnWholeSet, eval_PerBatch_OnWholeSet, eval_PerBatch_OnEachBatch = evaluation(
X_train, y_train, batch_size, batches, epochs, model, weights_dir)
mean_loss_per_epoch = mean_loss(eval_PerBatch_OnEachBatch, batches, epochs)
column_names = [
'log per epoch (train)', 'log per epoch (val)', 'learning rate',
'log per batch (train)', 'eval per epoch',
'eval per batch on whole training set',
'eval per batch on each corresponding batch',
'eval per batch on each corresponding batch (mean per epoch)'
]
loss_all = pd.concat([
history_PerEpoch[['loss', 'val_loss', 'lr']], history_PerBatch,
eval_PerEpoch_OnWholeSet, eval_PerBatch_OnWholeSet,
eval_PerBatch_OnEachBatch, mean_loss_per_epoch
],
axis=1,
ignore_index=False)
loss_all.columns = column_names
loss_all.to_csv(model_dir + '/loss_values.csv')
# prediction
y_pred = model.predict(X_train)
# plot
plot_loss(batches, epochs, column_names, loss_all, model_dir, model_name)
plot_prediction(y_pred, y_train, model_dir, model_name, batches)
# reset weights
keras.backend.clear_session()
return loss_all
y_test_predicted = y_test_p
#%%
for k in range(1, num_steps_to_predict):
x_encoder_test = y_test_p
y_test_p = model.predict([x_encoder_test, x_decoder_test])
y_test_predicted = np.hstack((y_test_predicted, y_test_p))
k = k + target_sequence_length
#%%
indices = np.random.choice(range(x_encoder_test.shape[0]),
replace=False,
size=10)
for index in indices:
plot_prediction(x_encoder_test[index, :, :], y_test[index, :, :],
y_test_predicted[index, :, :])
#%%
encoder_predict_model = keras.models.Model(encoder_inputs, encoder_states)
decoder_states_inputs = []
# Read layers backwards to fit the format of initial_state
# For some reason, the states of the model are order backwards (state of the first layer at the end of the list)
# If instead of a GRU you were using an LSTM Cell, you would have to append two Input tensors since the LSTM has 2 states.
for hidden_neurons in layers[::-1]:
# One state for GRU
decoder_states_inputs.append(keras.layers.Input(shape=(hidden_neurons, )))
decoder_outputs_and_states = decoder(decoder_inputs,
initial_state=decoder_states_inputs)
# uncomment to see model summary
#model.summary()
# Create model using optimizer Adam and loss function mean square error
model.compile(optimizer='adam', loss='mse')
# Checkpoint for saving the weights whenever validation loss improves
chkpnt = ModelCheckpoint(filepath='model.h5',
verbose=1,
save_best_only=True,
monitor='val_loss')
# Stop training when validation loss fails to decrease after 3 epochs
stop = EarlyStopping(monitor='val_loss', patience=3, verbose=1)
#############################################################
# Traninig Process #
#############################################################
try:
# Train model for a validation set of 20%.
model.fit(X_train,y_train, nb_epoch=epochs,verbose=1,batch_size=batch_size,\
shuffle=True,validation_split=0.2,\
callbacks=[ chkpnt, stop])
# Save model to file model.json
utils.save_model(model)
# Plot regression using testing set
utils.plot_prediction(model, X_test, y_test)
except KeyboardInterrupt:
# incase of interruption save model and plot regression
utils.save_model(model)
utils.plot_prediction(model, X_test, y_test)
test_folder = sys.argv[2]
suffix_name = Prediction.suffix_name
test_fileList = []
Prediction.getFileList(test_folder, test_fileList)
testSeries = list(map(load.loadFile, test_fileList))
testSeries, testEffDataLen = Prediction.align(testSeries)
test_input_data = np.asarray(testSeries, dtype=testSeries[0].dtype)
for seq_index in range(len(test_fileList)):
# Take one sequence (part of the training set)
# for trying out decoding.
input_seq = test_input_data[seq_index:seq_index + 1]
decoded_sentence = Prediction.predict(input_seq,
Prediction.encoder_predict_model,
Prediction.decoder_predict_model,
Prediction.num_steps_to_predict)
print('-')
input_seq[:, :, 0] *= 86400
input_seq[:, :, 1] *= 180
input_seq[:, :, 2] *= 360
print(input_seq)
print('pre:', decoded_sentence)
class_res = Classification.estimator.predict(decoded_sentence)
print('classification result:',
Classification.encoder.inverse_transform(class_res))
utils.plot_prediction(input_seq[0, :, :], None, decoded_sentence[0, :, :])
