def model_train(inputs, blocks, args, sum_path='./output/tensorboard',load_path='./output/models/',load=False):
'''
Train the base model.
:param inputs: instance of class Dataset, data source for training.
:param blocks: list, channel configs of st_conv blocks.
:param args: instance of class argparse, args for training.
'''
n, n_his, n_pred = args.n_route, args.n_his, args.n_pred
Ks, Kt = args.ks, args.kt
batch_size, epoch, inf_mode, opt = args.batch_size, args.epoch, args.inf_mode, args.opt
# Placeholder for model training
x = tf.placeholder(tf.float32, [None, n_his + n_pred, n, 1], name='data_input')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
# Define model loss
train_loss, pred = build_model(x, n_his, Ks, Kt, blocks, keep_prob)
tf.summary.scalar('train_loss', train_loss)
copy_loss = tf.add_n(tf.get_collection('copy_loss'))
tf.summary.scalar('copy_loss', copy_loss)
# Learning rate settings
global_steps = tf.Variable(0, trainable=False)
len_train = inputs.get_len('train')
if len_train % batch_size == 0:
epoch_step = len_train / batch_size
else:
epoch_step = int(len_train / batch_size) + 1
# Learning rate decay with rate 0.7 every 5 epochs.
lr = tf.train.exponential_decay(args.lr, global_steps, decay_steps=5 * epoch_step, decay_rate=0.7, staircase=True)
tf.summary.scalar('learning_rate', lr)
step_op = tf.assign_add(global_steps, 1)
with tf.control_dependencies([step_op]):
if opt == 'RMSProp':
train_op = tf.train.RMSPropOptimizer(lr).minimize(train_loss)
elif opt == 'ADAM':
train_op = tf.train.AdamOptimizer(lr).minimize(train_loss)
else:
raise ValueError(f'ERROR: optimizer "{opt}" is not defined.')
merged = tf.summary.merge_all()
##########################################################################
saver = tf.train.Saver(max_to_keep=3);
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
writer = tf.summary.FileWriter(pjoin(sum_path, 'train'), sess.graph)
if(load):
ckpt = tf.train.get_checkpoint_state(load_path)
saver.restore(sess, ckpt.model_checkpoint_path)
print(f'>> Loading saved model from {load_path} ...')
else:
sess.run(tf.global_variables_initializer())
if inf_mode == 'sep':
# for inference mode 'sep', the type of step index is int.
step_idx = n_pred - 1
tmp_idx = [step_idx]
min_val = min_va_val = np.array([4e1, 1e5, 1e5])
elif inf_mode == 'merge':
# for inference mode 'merge', the type of step index is np.ndarray.
# step_idx = tmp_idx = np.arange(3, n_pred + 1, 3) - 1
step_idx = tmp_idx = np.array([1,2,3,4,5,6,7,8])-1
min_val = min_va_val = np.array([4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5])
else:
raise ValueError(f'ERROR: test mode "{inf_mode}" is not defined.')
for i in range(epoch):
start_time = time.time()
for j, x_batch in enumerate(
gen_batch(inputs.get_data('train'), batch_size, dynamic_batch=True, shuffle=True)):
summary, _ = sess.run([merged, train_op], feed_dict={x: x_batch[:, 0:n_his + n_pred, :, :], keep_prob: 1.0})
writer.add_summary(summary, i * epoch_step + j)
if j % 50 == 0:
loss_value = \
sess.run([train_loss, copy_loss],
feed_dict={x: x_batch[:, 0:n_his + n_pred, :, :], keep_prob: 1.0})
print(f'Epoch {i:2d}, Step {j:3d}: [{loss_value[0]:.3f}, {loss_value[1]:.3f}]')
print(f'Epoch {i:2d} Training Time {time.time() - start_time:.3f}s')
if (i + 1) % args.save == 0:
model_save(sess, global_steps, 'STGCN')
# print('sleep begin')
# time.sleep(90)
# print('sleep end')
if((i+1)%5!=0):
continue
start_time = time.time()
min_va_val, min_val = \
model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val)
cnt=0;
for ix in tmp_idx:
# va, te = min_va_val[ix - 2:ix + 1], min_val[ix - 2:ix + 1]
va,te=min_va_val[cnt:cnt+3],min_val[cnt:cnt+3]
cnt+=3
print(f'Time Step {ix + 1}: '
f'MAPE {va[0]:7.3%}; '
f'MAE {va[1]:4.3f}; '
f'RMSE {va[2]:6.3f}.')
print(f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s')
writer.close()
print('Training model finished!')
def model_train(inputs,
blocks,
args,
sum_path='./output/tensorboard',
output_dim=1):
'''
Train the base model.
:param inputs: instance of class Dataset, data source for training.
:param blocks: list, channel configs of st_conv blocks.
:param args: instance of class argparse, args for training.
'''
n, n_his, n_pred = args.n_route, args.seq_len, args.horizon
Ks, Kt = args.ks, args.kt
batch_size, epoch, inf_mode, opt = args.batch_size, args.epoch, args.inf_mode, args.opt
# Placeholder for model training
x = tf.compat.v1.placeholder(tf.float32, [None, n_his + 1, n, 3],
name='data_input')
keep_prob = tf.compat.v1.placeholder(tf.float32, name='keep_prob')
# Define model loss, for one step forecasting...
train_loss, pred = build_model(x,
n_his,
Ks,
Kt,
blocks,
keep_prob,
output_dim=output_dim)
tf.summary.scalar('train_loss', train_loss)
# copy loss just using the previous step as current step prediction
copy_loss = tf.add_n(tf.get_collection('copy_loss'))
tf.summary.scalar('copy_loss', copy_loss)
# Learning rate settings
global_steps = tf.Variable(0, trainable=False)
len_train = inputs.get_len('train')
if len_train % batch_size == 0:
epoch_step = len_train / batch_size
else:
epoch_step = int(len_train / batch_size) + 1
# Learning rate decay with rate 0.7 every 5 epochs.
lr = tf.train.exponential_decay(args.lr,
global_steps,
decay_steps=5 * epoch_step,
decay_rate=0.7,
staircase=True)
tf.summary.scalar('learning_rate', lr)
step_op = tf.assign_add(global_steps, 1)
with tf.control_dependencies([step_op]):
if opt == 'RMSProp':
train_op = tf.train.RMSPropOptimizer(lr).minimize(train_loss)
elif opt == 'ADAM':
train_op = tf.train.AdamOptimizer(lr).minimize(train_loss)
else:
raise ValueError(f'ERROR: optimizer "{opt}" is not defined.')
merged = tf.summary.merge_all()
with tf.Session() as sess:
writer = tf.summary.FileWriter(pjoin(sum_path, 'train'), sess.graph)
sess.run(tf.global_variables_initializer())
if inf_mode == 'sep':
# for inference mode 'sep', the type of step index is int.
step_idx = n_pred - 1
tmp_idx = [step_idx]
min_val = min_va_val = np.array([4e1, 1e5, 1e5])
elif inf_mode == 'merge':
# for inference mode 'merge', the type of step index is np.ndarray.
# step_idx = tmp_idx = np.arange(3, n_pred + 1, 3) - 1
step_idx = tmp_idx = np.arange(n_pred)
min_val = min_va_val = np.array([4e1, 1e5, 1e5] * len(step_idx))
else:
raise ValueError(f'ERROR: test mode "{inf_mode}" is not defined.')
for i in range(epoch):
start_time = time.time()
for j, x_batch in enumerate(
gen_batch(inputs.get_data('train'),
batch_size,
dynamic_batch=True,
shuffle=True)):
summary, _ = sess.run([merged, train_op],
feed_dict={
x: x_batch[:, 0:n_his + 1, :, :],
keep_prob: 1.0
})
writer.add_summary(summary, i * epoch_step + j)
if j % 50 == 0:
loss_value = \
sess.run([train_loss, copy_loss],
feed_dict={x: x_batch[:, 0:n_his + 1, :, :], keep_prob: 1.0})
print(
f'Epoch {i:2d}, Step {j:3d}: [model_loss: {loss_value[0]:.3f}, copy_loss: {loss_value[1]:.3f}]',
flush=True)
# # for testing
# min_va_val, min_val = \
# model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val)
# for ix in tmp_idx:
# va, te = min_va_val[ix*3:(ix + 1)*3], min_val[ix*3:(ix + 1)*3]
# # va, te = min_va_val[ix], min_val[ix]
# print(f'Time Step {ix + 1}: '
# f'MAPE {va[0]:7.3%}, {te[0]:7.3%}; '
# f'MAE {va[1]:4.3f}, {te[1]:4.3f}; '
# f'RMSE {va[2]:6.3f}, {te[2]:6.3f}.', flush=True)
# print(f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s', flush=True)
print(
f'Epoch {i:2d} Training Time {time.time() - start_time:.3f}s')
start_time = time.time()
min_va_val, min_val = \
model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val)
for ix in tmp_idx:
va, te = min_va_val[ix * 3:(ix + 1) *
3], min_val[ix * 3:(ix + 1) * 3]
# va, te = min_va_val[ix], min_val[ix]
print(f'Time Step {ix + 1}: '
f'MAPE {va[0]:7.3%}, {te[0]:7.3%}; '
f'MAE {va[1]:4.3f}, {te[1]:4.3f}; '
f'RMSE {va[2]:6.3f}, {te[2]:6.3f}.')
print(
f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s')
if (i + 1) % args.save == 0:
model_save(sess, global_steps, 'STGCN')
writer.close()
print('Training model finished!')
def transient_model_inference(sess, pred, inputs, batch_size, n_his, n_pred,
step_idx, min_va_val, min_val, p_start_t, x_mean,
epoch, global_steps, model_feature,
val_pred_length, test_pred_length, scenario_data,
test_data):
'''
Model inference function.
:param sess: tf.Session().
:param pred: placeholder.
:param inputs: instance of class Dataset, data source for inference.
:param batch_size: int, the size of batch.
:param n_his: int, the length of historical records for training.
:param n_pred: int, the length of prediction.
:param step_idx: int or list, index for prediction slice.
:param min_va_val: np.ndarray, metric values on validation set.
:param min_val: np.ndarray, metric values on test set.
'''
min_val_copy = min_val.copy()
start_time = time.time()
print("Validation evaluation starts!!!")
x_val, x_test, x_stats = inputs.get_data('val'), inputs.get_data(
'test'), inputs.get_stats()
if n_his + 1 > x_val.shape[1]:
raise ValueError(
f'ERROR: the value of n_pred "{n_pred}" exceeds the length limit.')
val_predicion_array = []
val_rmse_array = []
prediction_length = val_pred_length
i_stop = int(prediction_length / n_pred)
if i_stop != prediction_length:
raise ValueError(
f'ERROR: the value of n_pred "{n_pred}" is not equal to 1.')
x_val_target = p_start_t - n_his
x_val_process = x_val[x_val_target - 1:x_val_target, :, :, :]
batch_size = 1
#need at least n_his round to make sure that all the prediction now is updated
temp_pred = n_his
y_pred, len_pred = multi_pred_process(sess, pred, x_val_process,
batch_size, n_his, temp_pred,
step_idx)
val_prediction = y_pred[:, 0, :, :]
val_predicion_array.append(val_prediction)
val_as_input_array = val_prediction.reshape(1, val_prediction.shape[0],
val_prediction.shape[1],
val_prediction.shape[2])
###evaluate n_his size prediction and ground truthe next n_his size
load_path = './output/models/'
#continue to process the rest of the remaining prediction nodes
i_remain = prediction_length - val_prediction.shape[0]
x_val_dummy = val_as_input_array[:, n_his - 1:n_his, :, :]
x_val_process = val_as_input_array
x_val_process = np.concatenate((x_val_process, x_val_dummy), axis=1)
for i in range(0, i_remain):
y_pred, len_pred = multi_pred(sess, pred, x_val_process, batch_size,
n_his, n_pred, step_idx)
#update x_val_process
x_val_process[:, 0:n_his - 1, :, :] = x_val_process[:, 1:n_his, :, :]
x_val_process[:, n_his - 1, :, :] = y_pred
val_prediction = y_pred[:, 0, :, :]
val_predicion_array[0] = np.concatenate(
(val_predicion_array[0], val_prediction), axis=0)
val_prediction_all = val_predicion_array[0]
#write to get the val_ground_truth
val_ground_truth_all_bak = scenario_data[p_start_t - 1:p_start_t +
val_pred_length - 1, :]
#print("verify the output")
#print(val_ground_truth_all_bak.shape)
#print(val_prediction_all.shape)
scenario = 'valid'
#draw figure
#draw_transient_actual_vs_prediction(val_prediction_all, x_stats, p_start_t, load_path, x_mean, epoch,scenario, val_ground_truth_all_bak)
print(f'Validation Inference Time {time.time() - start_time:.3f}s')
evl_val = transient_power_grid_evaluation(val_prediction_all, x_stats,
x_mean, val_ground_truth_all_bak)
print(
"###########Validation Performance Metrics Values are #######################"
)
print(f'MAPE validation {evl_val[0]:7.3%}')
print(f'MAE validation {evl_val[1]:14.8f}')
print(f'RMSE validation {evl_val[2]:6.3f}')
# chks: indicator that reflects the relationship of values between evl_val and min_va_val.
chks = evl_val < min_va_val
# update the metric on test set, if model's performance got improved on the validation.
#Always draw testing figures for now
if True:
#print("current epoch number is %s"%(epoch))
start_t_time = time.time()
#print("testing evaluation starts!!!")
min_va_val[chks] = evl_val[chks]
##starting the test part
test_predicion_array = []
test_rmse_array = []
prediction_length = test_pred_length
i_stop = int(prediction_length / n_pred)
if i_stop != prediction_length:
raise ValueError(
f'ERROR: the value of n_pred "{n_pred}" is not equal to 1.')
x_test_target = p_start_t - n_his
x_test_process = x_test[x_test_target - 1:x_test_target, :, :, :]
batch_size = 1
# need at least n_his round to make sure that all the prediction now is updated
temp_pred = n_his
y_test_pred, len_test_pred = multi_pred_process(
sess, pred, x_test_process, batch_size, n_his, temp_pred, step_idx)
test_prediction = y_test_pred[:, 0, :, :]
test_predicion_array.append(test_prediction)
test_as_input_array = test_prediction.reshape(1,
test_prediction.shape[0],
test_prediction.shape[1],
test_prediction.shape[2])
###evaluate n_his size prediction and ground truthe next n_his size
# draw the figures to show the difference between ground truth and prediction
load_path = './output/models/'
# continue to process the rest of the remaining prediction nodes
i_remain = prediction_length - test_prediction.shape[0]
x_test_dummy = test_as_input_array[:, n_his - 1:n_his, :, :]
x_test_process = test_as_input_array
x_test_process = np.concatenate((x_test_process, x_test_dummy), axis=1)
for i in range(0, i_remain):
y_test_pred, len_test_pred = multi_pred(sess, pred, x_test_process,
batch_size, n_his, n_pred,
step_idx)
# update x_val_process
x_test_process[:,
0:n_his - 1, :, :] = x_test_process[:,
1:n_his, :, :]
x_test_process[:, n_his - 1, :, :] = y_test_pred
test_prediction = y_test_pred[:, 0, :, :]
test_predicion_array[0] = np.concatenate(
(test_predicion_array[0], test_prediction), axis=0)
test_prediction_all = test_predicion_array[0]
inference_time = time.time() - start_t_time
#print(f'Testing Inference Time {inference_time:.3f}s')
#save the testing inference time to a file
with open(load_path + "inference_time.txt", mode='w') as file:
file.write('Testing Inference Time %s seconds.\n' %
(inference_time))
# write to get the test_ground_truth
test_ground_truth_all_bak = test_data[p_start_t - 1:p_start_t +
test_pred_length - 1, :]
'''
x_mean =1
scenario='test_valid'
draw_transient_actual_vs_prediction(test_prediction_all, x_stats, p_start_t, load_path, x_mean,
epoch,scenario, test_ground_truth_all_bak)
'''
evl_pred = transient_power_grid_evaluation(test_prediction_all,
x_stats, x_mean,
test_ground_truth_all_bak)
test_chks = evl_pred < min_val_copy
#print("###########tesing prediction values is")
#print(f'MAPE testing {evl_pred[0]:7.3%}')
#print(f'MAE testing {evl_pred[1]:14.8f}')
#print(f'RMSE testing {evl_pred[2]:6.3f}')
#save the model for each epoch
print("saving the model")
print('current epoch is %d' % epoch)
model_save(sess, global_steps, 'STGCN', n_his, n_pred, model_feature)
#using MAE as the indicator to save the model
if sum(test_chks):
#print("saving the model")
#print('current epoch is %d'%epoch)
#model_save(sess, global_steps, 'STGCN', n_his, n_pred,model_feature)
min_val_copy[test_chks] = evl_pred[test_chks]
return min_va_val, min_val_copy