# Change the path if you need other abnormal series. Be sure the series are
# stored in a format of list of arrays (shape = [n, 128]).
abnormal_series_list = []
print('Start retrieving abnormal series....')
for filename in sorted(glob.glob(abnormal_output_path + '*.txt')):
print(filename)
series = utils.txt_to_series(filename)
abnormal_series_list.append(series)
##########################################################
# 4. Construct MSE DataFrame for Full Anom Data
##########################################################
# Construct MSE DataFrame
anom_hat_list = [
utils.model_forecast(model, i, batch_size, window_size, predict_size,
shift_eval).reshape(-1, shift_eval, 128)
for i in abnormal_series_list
]
if shift_eval == predict_size + window_size:
anom_true_list = [
utils.windowed_true(i, shift_eval, predict_size)
for i in abnormal_series_list
]
else:
anom_true_list = [
i[window_size:, :].reshape((-1, shift_eval, 128))
for i in abnormal_series_list
]
##########################################################
print(filename)
series = utils.txt_to_series(filename)[:, 87:123]
abnormal_series_list.append(series)
# Comment out the following operation if you do not need validation data
with open(full_x_valid_filename, 'rb') as f:
full_x_valid = joblib.load(f)
# Comment out the next section if you do not need validation data
##########################################################
# 3. Construct MSE DataFrame for Validation Data
##########################################################
print('Start constructing mse DataFrame...')
# Get valid_hat and evaluate time
start = timer()
valid_hat = utils.model_forecast(model, full_x_valid, batch_size, window_size,
predict_size, shift_eval)
end = timer()
validation_time = (start - end) / (len(full_x_valid) // shift_eval)
print('Validation spends {} seconds! Hmm...'.format(validation_time))
# Get valid true
valid_hat_dim = np.shape(valid_hat)[0]
valid_true = np.reshape(
utils.windowed_true_control(full_x_valid, shift_eval, predict_size),
(-1, predict_size, 36))
# Create mse DataFrame
valid_mse = np.mean(np.power(valid_hat - valid_true[:valid_hat_dim], 2),
axis=(1, 2))
valid_error_df = pd.DataFrame({'valid_error': valid_mse})
abnormal_series_list.append(series)
break
# Comment out the following operation if you do not need validation data
with open(full_x_valid_filename, 'rb') as f:
full_x_valid = joblib.load(f)
# Comment out the next section if you do not need validation data
##########################################################
# 3. Construct MSE DataFrame for Validation Data
##########################################################
print('Start constructing mse DataFrame...')
# Get valid_hat and evaluate time
start = timer()
valid_hat = utils.model_forecast(model, full_x_valid, batch_size, window_size,
predict_size,
shift_eval).reshape(-1, shift_eval, 128)
end = timer()
validation_time = (start - end) / (len(full_x_valid) // shift_eval)
print('Validation spends {} seconds! Hmm...'.format(validation_time))
# Get valid true
if shift_eval == predict_size + window_size:
valid_true = utils.windowed_true(full_x_valid, shift_eval, predict_size)
else:
valid_true = full_x_valid[window_size:, :].reshape((-1, shift_eval, 128))
# Create mse DataFrame
valid_mse = np.mean(np.power(valid_hat - valid_true, 2), axis=(1, 2))
valid_error_df = pd.DataFrame({'valid_error': valid_mse})
window_size = 20
batch_size = 32
shuffle_buffer_size = 1000
# plot_series(time, series, title = "Original Data")
# plt.show()
print("\n Please be patient! _()_ This might take some time. \n")
# forecast = []
# for time in range(len(series) - window_size):
# forecast.append(model.predict(series[time:time + window_size][np.newaxis]))
# forecast = forecast[split_time-window_size:]
# results = np.array(forecast)[:, 0, 0]
rnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)
rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]
plt.figure(figsize=(10, 6))
plot_series(time_valid, X_valid)
plot_series(
time_valid,
rnn_forecast,
title="conv_lstm prediction",
text="Conv1D(32)\nLSTM(32)\nLSTM(32)\nDense(1)\nloss = Huber\nOptimizer=SGD"
)
plt.show()
# plt.savefig('plotted_graphs/simple_model.png', bbox_inches='tight')
print(filename)
series = utils.txt_to_series(filename)
abnormal_series_list.append(series)
# Comment out the following operation if you do not need validation data
with open(full_x_valid_filename, 'rb') as f:
full_x_valid = joblib.load(f)
# Comment out the next section if you do not need validation data
##########################################################
# 3. Construct MSE DataFrame for Validation Data
##########################################################
print('Start constructing mse DataFrame...')
# Get valid_hat and evaluate time
start = timer()
valid_hat = utils.model_forecast(model, full_x_valid, batch_size, window_size,
predict_size, shift_eval).reshape(-1, shift_eval, 128)
end = timer()
validation_time = (start - end) / (len(full_x_valid) // shift_eval)
print('Validation spends {} seconds! Hmm...'.format(validation_time))
# Get valid true
if shift_eval == predict_size + window_size:
valid_true = utils.windowed_true(full_x_valid, shift_eval, predict_size).reshape(-1, shift_eval, 128)
else:
valid_true = full_x_valid[window_size:, :].reshape(-1, shift_eval, 128)
# Create mse DataFrame
valid_mse = np.mean(np.power(valid_hat - valid_true, 2), axis=(1, 2))
valid_error_df = pd.DataFrame({'valid_error': valid_mse})
# Save MSE DataFrame