def evaluate(model, loss_fn, test_loader, params, dirs, istest=False):
'''Evaluate the model on the test set.
Args:
model: (torch.nn.Module) the Deep AR model
loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch
test_loader: load test data and labels
params: (Params) hyperparameters
'''
logger = logging.getLogger('DeepAR.Eval')
model.eval()
with torch.no_grad():
summary = {}
metrics = utils.init_metrics(params, dirs)
for i, (test_batch, labels) in enumerate(tqdm(test_loader)):
test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to(
dirs.device)
labels = labels.to(torch.float32).to(dirs.device)
batch_size = test_batch.shape[1]
hidden = model.init_hidden(batch_size)
cell = model.init_cell(batch_size)
for t in range(params.pred_start):
_, hidden, cell = model(test_batch[t].unsqueeze(0), hidden,
cell)
#save some params of SQF for plot
if istest and (i == 0):
plot_param, _, _ = model(
test_batch[params.pred_start].unsqueeze(0), hidden, cell)
save_name = os.path.join(dirs.model_dir, 'sqf_param')
with open(save_name, 'wb') as f:
import pickle
pickle.dump(plot_param, f)
pickle.dump(test_batch[params.pred_start], f)
samples, _, _ = model.predict(test_batch,
hidden,
cell,
sampling=True)
metrics = utils.update_metrics(metrics, samples, labels,
params.pred_start)
summary = utils.final_metrics(metrics)
if istest == False:
strings ='\nCRPS: '+str(summary['CRPS'])+\
'\nmre:'+str(summary['mre'].abs().max(dim=1)[0].mean().item())+\
'\nPINAW:'+str(summary['pinaw'].item())
logger.info('- Full test metrics: ' + strings)
else:
logger.info(' - Test Set CRPS: ' +
str(summary['CRPS'].mean().item()))
ss_metric = {}
ss_metric['CRPS_Mean'] = summary['CRPS'].mean()
ss_metric['mre'] = summary['mre'].abs().mean()
ss_metric['pinaw'] = summary['pinaw']
for i, crps in enumerate(summary['CRPS']):
ss_metric[f'CRPS_{i}'] = crps
for i, mre in enumerate(summary['mre'].mean(dim=0)):
ss_metric[f'mre_{i}'] = mre
return ss_metric
def evaluate(model, test_loader, params, plot_num):
'''Evaluate the model on the test set.
Args:
model: (torch.nn.Module) the Deep AR model
loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch
test_loader: load test data and labels
params: (Params) hyperparameters
plot_num: (-1): evaluation from evaluate.py; else (epoch): evaluation on epoch
sample: (boolean) do ancestral sampling or directly use output mu from last time step
'''
model.eval()
with torch.no_grad():
plot_batch = np.random.randint(len(test_loader) - 1)
summary_metric = {}
raw_metrics = utils.init_metrics()
sum_mu = torch.zeros([740, params.predict_steps]).to(params.device)
sum_sigma = torch.zeros([740, params.predict_steps]).to(params.device)
true = torch.zeros([740, params.predict_steps]).to(params.device)
for i, (test_batch, id_batch, v,
labels) in enumerate(tqdm(test_loader)):
test_batch = test_batch.to(torch.float32).to(params.device)
id_batch = id_batch.unsqueeze(-1).to(params.device)
v_batch = v.to(torch.float32).to(params.device)
labels = labels.to(torch.float32).to(params.device)
batch_size = test_batch.shape[0]
sample_mu, sample_q90 = transformer.test(model, params, test_batch,
v_batch, id_batch)
raw_metrics = utils.update_metrics(
raw_metrics,
sample_mu,
labels,
params.test_predict_start,
relative=params.relative_metrics)
if (i == 0):
sum_mu = sample_mu
sum_q90 = sample_q90
true = labels[:, -params.predict_steps:]
else:
sum_mu = torch.cat([sum_mu, sample_mu], 0)
sum_q90 = torch.cat([sum_q90, sample_q90], 0)
true = torch.cat([true, labels[:, -params.predict_steps:]], 0)
summary_metric = utils.final_metrics(raw_metrics)
summary_metric['q50'] = transformer.quantile_loss(0.5, sum_mu, true)
summary_metric['q90'] = transformer.quantile_loss(0.5, sum_q90, true)
summary_metric['MAPE'] = transformer.MAPE(sum_mu, true)
metrics_string = '; '.join('{}: {:05.3f}'.format(k, v)
for k, v in summary_metric.items())
logger.info('- Full test metrics: ' + metrics_string)
return summary_metric
def evaluate(model, loss_fn, test_loader, params, plot_num, sample=True):
'''Evaluate the model on the test set.
Args:
model: (torch.nn.Module) the Deep AR model
loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch
test_loader: load test data and labels
params: (Params) hyperparameters
plot_num: (-1): evaluation from evaluate.py; else (epoch): evaluation on epoch
sample: (boolean) do ancestral sampling or directly use output mu from last time step
'''
model.eval()
with torch.no_grad():
plot_batch = np.random.randint(len(test_loader)-1)
summary_metric = {}
raw_metrics = utils.init_metrics(sample=sample)
# Test_loader:
# test_batch ([batch_size, train_window, 1+cov_dim]): z_{0:T-1} + x_{1:T}, note that z_0 = 0;
# id_batch ([batch_size]): one integer denoting the time series id;
# v ([batch_size, 2]): scaling factor for each window;
# labels ([batch_size, train_window]): z_{1:T}.
for i, (test_batch, id_batch, v, labels) in enumerate(tqdm(test_loader)):
test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to(params.device)
id_batch = id_batch.unsqueeze(0).to(params.device)
v_batch = v.to(torch.float32).to(params.device)
labels = labels.to(torch.float32).to(params.device)
batch_size = test_batch.shape[1]
input_mu = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled
input_sigma = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled
hidden = model.init_hidden(batch_size)
cell = model.init_cell(batch_size)
for t in range(params.test_predict_start):
# if z_t is missing, replace it by output mu from the last time step
zero_index = (test_batch[t,:,0] == 0)
if t > 0 and torch.sum(zero_index) > 0:
test_batch[t,zero_index,0] = mu[zero_index]
mu, sigma, hidden, cell = model(test_batch[t].unsqueeze(0), id_batch, hidden, cell)
input_mu[:,t] = v_batch[:, 0] * mu + v_batch[:, 1]
input_sigma[:,t] = v_batch[:, 0] * sigma
if sample:
samples, sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell, sampling=True)
raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, samples, relative = params.relative_metrics)
else:
sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell)
raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, relative = params.relative_metrics)
if i == plot_batch:
if sample:
sample_metrics = utils.get_metrics(sample_mu, labels, params.test_predict_start, samples, relative = params.relative_metrics)
else:
sample_metrics = utils.get_metrics(sample_mu, labels, params.test_predict_start, relative = params.relative_metrics)
# select 10 from samples with highest error and 10 from the rest
top_10_nd_sample = (-sample_metrics['ND']).argsort()[:batch_size // 10] # hard coded to be 10
chosen = set(top_10_nd_sample.tolist())
all_samples = set(range(batch_size))
not_chosen = np.asarray(list(all_samples - chosen))
if batch_size < 100: # make sure there are enough unique samples to choose top 10 from
random_sample_10 = np.random.choice(top_10_nd_sample, size=10, replace=True)
else:
random_sample_10 = np.random.choice(top_10_nd_sample, size=10, replace=False)
if batch_size < 12: # make sure there are enough unique samples to choose bottom 90 from
random_sample_90 = np.random.choice(not_chosen, size=10, replace=True)
else:
random_sample_90 = np.random.choice(not_chosen, size=10, replace=False)
combined_sample = np.concatenate((random_sample_10, random_sample_90))
label_plot = labels[combined_sample].data.cpu().numpy()
predict_mu = sample_mu[combined_sample].data.cpu().numpy()
predict_sigma = sample_sigma[combined_sample].data.cpu().numpy()
plot_mu = np.concatenate((input_mu[combined_sample].data.cpu().numpy(), predict_mu), axis=1)
plot_sigma = np.concatenate((input_sigma[combined_sample].data.cpu().numpy(), predict_sigma), axis=1)
plot_metrics = {_k: _v[combined_sample] for _k, _v in sample_metrics.items()}
plot_eight_windows(params.plot_dir, plot_mu, plot_sigma, label_plot, params.test_window, params.test_predict_start, plot_num, plot_metrics, sample)
summary_metric = utils.final_metrics(raw_metrics, sampling=sample)
metrics_string = '; '.join('{}: {:05.3f}'.format(k, v) for k, v in summary_metric.items())
logger.info('- Full test metrics: ' + metrics_string)
return summary_metric
def evaluate(model, loss_fn, test_loader, params, sample=True):
'''
Evaluate the model on the test set.
Args:
model: (torch.nn.Module) the Deep AR model
loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch
test_loader: load test data and labels
params: (Params) hyperparameters
sample: (boolean) do ancestral sampling or directly use output mu from last time step
'''
model.eval()
with torch.no_grad():
summary_metric = {}
raw_metrics = utils.init_metrics(sample=sample)
# Test_loader:
# test_batch ([batch_size, train_window, 1+cov_dim]): z_{0:T-1} + x_{1:T}, note that z_0 = 0;
# id_batch ([batch_size]): one integer denoting the time series id;
# v ([batch_size, 2]): scaling factor for each window;
# labels ([batch_size, train_window]): z_{1:T}.
result_mu = []
result_sigma = []
for i, (test_batch, id_batch, v, labels) in enumerate(test_loader):
test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to(params.device)
id_batch = id_batch.unsqueeze(0).to(params.device)
v_batch = v.to(torch.float32).to(params.device)
labels = labels.to(torch.float32).to(params.device)
batch_size = test_batch.shape[1]
input_mu = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled
input_sigma = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled
hidden = model.init_hidden(batch_size)
cell = model.init_cell(batch_size)
for t in range(params.test_predict_start): # 先计算encoder部分
# if z_t is missing, replace it by output mu from the last time step
# 如果z_t缺失，用前一步预测值代替真实值作为输入
zero_index = (test_batch[t, :, 0] == 0)
if t > 0 and torch.sum(zero_index) > 0:
test_batch[t, zero_index, 0] = mu[zero_index]
mu, sigma, hidden, cell = model(test_batch[t].unsqueeze(0), id_batch, hidden, cell)
input_mu[:, t] = v_batch[:, 0] * mu + v_batch[:, 1] # v_batch[:, 1] == 0, useless
input_sigma[:, t] = v_batch[:, 0] * sigma
if not params.one_step:
test_batch[params.test_predict_start, :, 0] = mu
# 计算decoder部分
if sample:
samples, sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell, sampling=True, one_step=params.one_step)
raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, samples, relative = params.relative_metrics)
else:
sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell, one_step=params.one_step)
raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, relative = params.relative_metrics)
result_mu.append(sample_mu)
result_sigma.append(sample_sigma)
summary_metric = utils.final_metrics(raw_metrics, sampling=sample)
metrics_string = '; '.join('{}: {:05.3f}'.format(k, v) for k, v in summary_metric.items())
# print('test metrics: ' + metrics_string)
return summary_metric, result_mu, result_sigma