def model_train(inputs, blocks, args, sum_path='./output/tensorboard',load_path='./output/models/',load=False): ''' Train the base model. :param inputs: instance of class Dataset, data source for training. :param blocks: list, channel configs of st_conv blocks. :param args: instance of class argparse, args for training. ''' n, n_his, n_pred = args.n_route, args.n_his, args.n_pred Ks, Kt = args.ks, args.kt batch_size, epoch, inf_mode, opt = args.batch_size, args.epoch, args.inf_mode, args.opt # Placeholder for model training x = tf.placeholder(tf.float32, [None, n_his + n_pred, n, 1], name='data_input') keep_prob = tf.placeholder(tf.float32, name='keep_prob') # Define model loss train_loss, pred = build_model(x, n_his, Ks, Kt, blocks, keep_prob) tf.summary.scalar('train_loss', train_loss) copy_loss = tf.add_n(tf.get_collection('copy_loss')) tf.summary.scalar('copy_loss', copy_loss) # Learning rate settings global_steps = tf.Variable(0, trainable=False) len_train = inputs.get_len('train') if len_train % batch_size == 0: epoch_step = len_train / batch_size else: epoch_step = int(len_train / batch_size) + 1 # Learning rate decay with rate 0.7 every 5 epochs. lr = tf.train.exponential_decay(args.lr, global_steps, decay_steps=5 * epoch_step, decay_rate=0.7, staircase=True) tf.summary.scalar('learning_rate', lr) step_op = tf.assign_add(global_steps, 1) with tf.control_dependencies([step_op]): if opt == 'RMSProp': train_op = tf.train.RMSPropOptimizer(lr).minimize(train_loss) elif opt == 'ADAM': train_op = tf.train.AdamOptimizer(lr).minimize(train_loss) else: raise ValueError(f'ERROR: optimizer "{opt}" is not defined.') merged = tf.summary.merge_all() ########################################################################## saver = tf.train.Saver(max_to_keep=3); config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: writer = tf.summary.FileWriter(pjoin(sum_path, 'train'), sess.graph) if(load): ckpt = tf.train.get_checkpoint_state(load_path) saver.restore(sess, ckpt.model_checkpoint_path) print(f'>> Loading saved model from {load_path} ...') else: sess.run(tf.global_variables_initializer()) if inf_mode == 'sep': # for inference mode 'sep', the type of step index is int. step_idx = n_pred - 1 tmp_idx = [step_idx] min_val = min_va_val = np.array([4e1, 1e5, 1e5]) elif inf_mode == 'merge': # for inference mode 'merge', the type of step index is np.ndarray. # step_idx = tmp_idx = np.arange(3, n_pred + 1, 3) - 1 step_idx = tmp_idx = np.array([1,2,3,4,5,6,7,8])-1 min_val = min_va_val = np.array([4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5,4e1, 1e5, 1e5]) else: raise ValueError(f'ERROR: test mode "{inf_mode}" is not defined.') for i in range(epoch): start_time = time.time() for j, x_batch in enumerate( gen_batch(inputs.get_data('train'), batch_size, dynamic_batch=True, shuffle=True)): summary, _ = sess.run([merged, train_op], feed_dict={x: x_batch[:, 0:n_his + n_pred, :, :], keep_prob: 1.0}) writer.add_summary(summary, i * epoch_step + j) if j % 50 == 0: loss_value = \ sess.run([train_loss, copy_loss], feed_dict={x: x_batch[:, 0:n_his + n_pred, :, :], keep_prob: 1.0}) print(f'Epoch {i:2d}, Step {j:3d}: [{loss_value[0]:.3f}, {loss_value[1]:.3f}]') print(f'Epoch {i:2d} Training Time {time.time() - start_time:.3f}s') if (i + 1) % args.save == 0: model_save(sess, global_steps, 'STGCN') # print('sleep begin') # time.sleep(90) # print('sleep end') if((i+1)%5!=0): continue start_time = time.time() min_va_val, min_val = \ model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val) cnt=0; for ix in tmp_idx: # va, te = min_va_val[ix - 2:ix + 1], min_val[ix - 2:ix + 1] va,te=min_va_val[cnt:cnt+3],min_val[cnt:cnt+3] cnt+=3 print(f'Time Step {ix + 1}: ' f'MAPE {va[0]:7.3%}; ' f'MAE {va[1]:4.3f}; ' f'RMSE {va[2]:6.3f}.') print(f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s') writer.close() print('Training model finished!')
def model_train(inputs, blocks, args, sum_path='./output/tensorboard', output_dim=1): ''' Train the base model. :param inputs: instance of class Dataset, data source for training. :param blocks: list, channel configs of st_conv blocks. :param args: instance of class argparse, args for training. ''' n, n_his, n_pred = args.n_route, args.seq_len, args.horizon Ks, Kt = args.ks, args.kt batch_size, epoch, inf_mode, opt = args.batch_size, args.epoch, args.inf_mode, args.opt # Placeholder for model training x = tf.compat.v1.placeholder(tf.float32, [None, n_his + 1, n, 3], name='data_input') keep_prob = tf.compat.v1.placeholder(tf.float32, name='keep_prob') # Define model loss, for one step forecasting... train_loss, pred = build_model(x, n_his, Ks, Kt, blocks, keep_prob, output_dim=output_dim) tf.summary.scalar('train_loss', train_loss) # copy loss just using the previous step as current step prediction copy_loss = tf.add_n(tf.get_collection('copy_loss')) tf.summary.scalar('copy_loss', copy_loss) # Learning rate settings global_steps = tf.Variable(0, trainable=False) len_train = inputs.get_len('train') if len_train % batch_size == 0: epoch_step = len_train / batch_size else: epoch_step = int(len_train / batch_size) + 1 # Learning rate decay with rate 0.7 every 5 epochs. lr = tf.train.exponential_decay(args.lr, global_steps, decay_steps=5 * epoch_step, decay_rate=0.7, staircase=True) tf.summary.scalar('learning_rate', lr) step_op = tf.assign_add(global_steps, 1) with tf.control_dependencies([step_op]): if opt == 'RMSProp': train_op = tf.train.RMSPropOptimizer(lr).minimize(train_loss) elif opt == 'ADAM': train_op = tf.train.AdamOptimizer(lr).minimize(train_loss) else: raise ValueError(f'ERROR: optimizer "{opt}" is not defined.') merged = tf.summary.merge_all() with tf.Session() as sess: writer = tf.summary.FileWriter(pjoin(sum_path, 'train'), sess.graph) sess.run(tf.global_variables_initializer()) if inf_mode == 'sep': # for inference mode 'sep', the type of step index is int. step_idx = n_pred - 1 tmp_idx = [step_idx] min_val = min_va_val = np.array([4e1, 1e5, 1e5]) elif inf_mode == 'merge': # for inference mode 'merge', the type of step index is np.ndarray. # step_idx = tmp_idx = np.arange(3, n_pred + 1, 3) - 1 step_idx = tmp_idx = np.arange(n_pred) min_val = min_va_val = np.array([4e1, 1e5, 1e5] * len(step_idx)) else: raise ValueError(f'ERROR: test mode "{inf_mode}" is not defined.') for i in range(epoch): start_time = time.time() for j, x_batch in enumerate( gen_batch(inputs.get_data('train'), batch_size, dynamic_batch=True, shuffle=True)): summary, _ = sess.run([merged, train_op], feed_dict={ x: x_batch[:, 0:n_his + 1, :, :], keep_prob: 1.0 }) writer.add_summary(summary, i * epoch_step + j) if j % 50 == 0: loss_value = \ sess.run([train_loss, copy_loss], feed_dict={x: x_batch[:, 0:n_his + 1, :, :], keep_prob: 1.0}) print( f'Epoch {i:2d}, Step {j:3d}: [model_loss: {loss_value[0]:.3f}, copy_loss: {loss_value[1]:.3f}]', flush=True) # # for testing # min_va_val, min_val = \ # model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val) # for ix in tmp_idx: # va, te = min_va_val[ix*3:(ix + 1)*3], min_val[ix*3:(ix + 1)*3] # # va, te = min_va_val[ix], min_val[ix] # print(f'Time Step {ix + 1}: ' # f'MAPE {va[0]:7.3%}, {te[0]:7.3%}; ' # f'MAE {va[1]:4.3f}, {te[1]:4.3f}; ' # f'RMSE {va[2]:6.3f}, {te[2]:6.3f}.', flush=True) # print(f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s', flush=True) print( f'Epoch {i:2d} Training Time {time.time() - start_time:.3f}s') start_time = time.time() min_va_val, min_val = \ model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val) for ix in tmp_idx: va, te = min_va_val[ix * 3:(ix + 1) * 3], min_val[ix * 3:(ix + 1) * 3] # va, te = min_va_val[ix], min_val[ix] print(f'Time Step {ix + 1}: ' f'MAPE {va[0]:7.3%}, {te[0]:7.3%}; ' f'MAE {va[1]:4.3f}, {te[1]:4.3f}; ' f'RMSE {va[2]:6.3f}, {te[2]:6.3f}.') print( f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s') if (i + 1) % args.save == 0: model_save(sess, global_steps, 'STGCN') writer.close() print('Training model finished!')
def transient_model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val, p_start_t, x_mean, epoch, global_steps, model_feature, val_pred_length, test_pred_length, scenario_data, test_data): ''' Model inference function. :param sess: tf.Session(). :param pred: placeholder. :param inputs: instance of class Dataset, data source for inference. :param batch_size: int, the size of batch. :param n_his: int, the length of historical records for training. :param n_pred: int, the length of prediction. :param step_idx: int or list, index for prediction slice. :param min_va_val: np.ndarray, metric values on validation set. :param min_val: np.ndarray, metric values on test set. ''' min_val_copy = min_val.copy() start_time = time.time() print("Validation evaluation starts!!!") x_val, x_test, x_stats = inputs.get_data('val'), inputs.get_data( 'test'), inputs.get_stats() if n_his + 1 > x_val.shape[1]: raise ValueError( f'ERROR: the value of n_pred "{n_pred}" exceeds the length limit.') val_predicion_array = [] val_rmse_array = [] prediction_length = val_pred_length i_stop = int(prediction_length / n_pred) if i_stop != prediction_length: raise ValueError( f'ERROR: the value of n_pred "{n_pred}" is not equal to 1.') x_val_target = p_start_t - n_his x_val_process = x_val[x_val_target - 1:x_val_target, :, :, :] batch_size = 1 #need at least n_his round to make sure that all the prediction now is updated temp_pred = n_his y_pred, len_pred = multi_pred_process(sess, pred, x_val_process, batch_size, n_his, temp_pred, step_idx) val_prediction = y_pred[:, 0, :, :] val_predicion_array.append(val_prediction) val_as_input_array = val_prediction.reshape(1, val_prediction.shape[0], val_prediction.shape[1], val_prediction.shape[2]) ###evaluate n_his size prediction and ground truthe next n_his size load_path = './output/models/' #continue to process the rest of the remaining prediction nodes i_remain = prediction_length - val_prediction.shape[0] x_val_dummy = val_as_input_array[:, n_his - 1:n_his, :, :] x_val_process = val_as_input_array x_val_process = np.concatenate((x_val_process, x_val_dummy), axis=1) for i in range(0, i_remain): y_pred, len_pred = multi_pred(sess, pred, x_val_process, batch_size, n_his, n_pred, step_idx) #update x_val_process x_val_process[:, 0:n_his - 1, :, :] = x_val_process[:, 1:n_his, :, :] x_val_process[:, n_his - 1, :, :] = y_pred val_prediction = y_pred[:, 0, :, :] val_predicion_array[0] = np.concatenate( (val_predicion_array[0], val_prediction), axis=0) val_prediction_all = val_predicion_array[0] #write to get the val_ground_truth val_ground_truth_all_bak = scenario_data[p_start_t - 1:p_start_t + val_pred_length - 1, :] #print("verify the output") #print(val_ground_truth_all_bak.shape) #print(val_prediction_all.shape) scenario = 'valid' #draw figure #draw_transient_actual_vs_prediction(val_prediction_all, x_stats, p_start_t, load_path, x_mean, epoch,scenario, val_ground_truth_all_bak) print(f'Validation Inference Time {time.time() - start_time:.3f}s') evl_val = transient_power_grid_evaluation(val_prediction_all, x_stats, x_mean, val_ground_truth_all_bak) print( "###########Validation Performance Metrics Values are #######################" ) print(f'MAPE validation {evl_val[0]:7.3%}') print(f'MAE validation {evl_val[1]:14.8f}') print(f'RMSE validation {evl_val[2]:6.3f}') # chks: indicator that reflects the relationship of values between evl_val and min_va_val. chks = evl_val < min_va_val # update the metric on test set, if model's performance got improved on the validation. #Always draw testing figures for now if True: #print("current epoch number is %s"%(epoch)) start_t_time = time.time() #print("testing evaluation starts!!!") min_va_val[chks] = evl_val[chks] ##starting the test part test_predicion_array = [] test_rmse_array = [] prediction_length = test_pred_length i_stop = int(prediction_length / n_pred) if i_stop != prediction_length: raise ValueError( f'ERROR: the value of n_pred "{n_pred}" is not equal to 1.') x_test_target = p_start_t - n_his x_test_process = x_test[x_test_target - 1:x_test_target, :, :, :] batch_size = 1 # need at least n_his round to make sure that all the prediction now is updated temp_pred = n_his y_test_pred, len_test_pred = multi_pred_process( sess, pred, x_test_process, batch_size, n_his, temp_pred, step_idx) test_prediction = y_test_pred[:, 0, :, :] test_predicion_array.append(test_prediction) test_as_input_array = test_prediction.reshape(1, test_prediction.shape[0], test_prediction.shape[1], test_prediction.shape[2]) ###evaluate n_his size prediction and ground truthe next n_his size # draw the figures to show the difference between ground truth and prediction load_path = './output/models/' # continue to process the rest of the remaining prediction nodes i_remain = prediction_length - test_prediction.shape[0] x_test_dummy = test_as_input_array[:, n_his - 1:n_his, :, :] x_test_process = test_as_input_array x_test_process = np.concatenate((x_test_process, x_test_dummy), axis=1) for i in range(0, i_remain): y_test_pred, len_test_pred = multi_pred(sess, pred, x_test_process, batch_size, n_his, n_pred, step_idx) # update x_val_process x_test_process[:, 0:n_his - 1, :, :] = x_test_process[:, 1:n_his, :, :] x_test_process[:, n_his - 1, :, :] = y_test_pred test_prediction = y_test_pred[:, 0, :, :] test_predicion_array[0] = np.concatenate( (test_predicion_array[0], test_prediction), axis=0) test_prediction_all = test_predicion_array[0] inference_time = time.time() - start_t_time #print(f'Testing Inference Time {inference_time:.3f}s') #save the testing inference time to a file with open(load_path + "inference_time.txt", mode='w') as file: file.write('Testing Inference Time %s seconds.\n' % (inference_time)) # write to get the test_ground_truth test_ground_truth_all_bak = test_data[p_start_t - 1:p_start_t + test_pred_length - 1, :] ''' x_mean =1 scenario='test_valid' draw_transient_actual_vs_prediction(test_prediction_all, x_stats, p_start_t, load_path, x_mean, epoch,scenario, test_ground_truth_all_bak) ''' evl_pred = transient_power_grid_evaluation(test_prediction_all, x_stats, x_mean, test_ground_truth_all_bak) test_chks = evl_pred < min_val_copy #print("###########tesing prediction values is") #print(f'MAPE testing {evl_pred[0]:7.3%}') #print(f'MAE testing {evl_pred[1]:14.8f}') #print(f'RMSE testing {evl_pred[2]:6.3f}') #save the model for each epoch print("saving the model") print('current epoch is %d' % epoch) model_save(sess, global_steps, 'STGCN', n_his, n_pred, model_feature) #using MAE as the indicator to save the model if sum(test_chks): #print("saving the model") #print('current epoch is %d'%epoch) #model_save(sess, global_steps, 'STGCN', n_his, n_pred,model_feature) min_val_copy[test_chks] = evl_pred[test_chks] return min_va_val, min_val_copy