def ev_test(self, name):
self.model.eval()
y_preds = []
y_truths = []
for iter, (x, y) in enumerate(self.dataloader[name+'_loader'].get_iterator()):
x, y = self._prepare_data(x, y)
with torch.no_grad():
preds = self.model(x,y)
y_preds.append(preds)
y = y[..., self.params['out_level']].squeeze()
y_truths.append(y)
y_preds = torch.cat(y_preds, axis=1)
y_truths = torch.cat(y_truths, axis=1)
mae = []
mape = []
rmse = []
for i in range(self.seq_out_len):
pred = self.scaler.inverse_transform(y_preds[i,...],self.params['out_level'])
real = self.scaler.inverse_transform(y_truths[i,...],self.params['out_level'])
results = metrics.metric(pred, real)
log = 'Evaluate best model on test data for horizon {:d}, Test MAE: {:.4f}, Test MAPE: {:.4f}, Test RMSE: {:.4f}'
print(log.format(i + 1, results[0], results[1], results[2]))
mae.append(results[0])
mape.append(results[1])
rmse.append(results[2])
return mae, mape, rmse
def test(self, setting):
test_data, test_loader = self._get_data(flag='test')
self.model.eval()
preds = []
trues = []
for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(test_loader):
batch_x = batch_x.float().to(self.device)
batch_y = batch_y.float()
batch_x_mark = batch_x_mark.float().to(self.device)
batch_y_mark = batch_y_mark.float().to(self.device)
# decoder input
dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).float()
dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).float().to(self.device)
# encoder - decoder
if self.args.use_amp:
with torch.cuda.amp.autocast():
if self.args.output_attention:
outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]
else:
outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)
else:
if self.args.output_attention:
outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]
else:
outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)
if self.args.inverse:
outputs = test_data.inverse_transform(outputs)
f_dim = -1 if self.args.features=='MS' else 0
batch_y = batch_y[:,-self.args.pred_len:,f_dim:].to(self.device)
pred = outputs.detach().cpu().numpy()#.squeeze()
true = batch_y.detach().cpu().numpy()#.squeeze()
preds.append(pred)
trues.append(true)
preds = np.array(preds)
trues = np.array(trues)
print('test shape:', preds.shape, trues.shape)
preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1])
print('test shape:', preds.shape, trues.shape)
# result save
folder_path = self.args.results_path + setting +'/'
if not os.path.exists(folder_path):
os.makedirs(folder_path)
mae, mse, rmse, mape, mspe = metric(preds, trues)
print('mse:{}, mae:{}'.format(mse, mae))
np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe]))
np.save(folder_path+'pred.npy', preds)
np.save(folder_path+'true.npy', trues)
return
def ev_valid(self, name):
self.model.eval()
outputs = []
realy = []
dummy = torch.zeros(10).requires_grad_()
for iter, (x, y) in enumerate(
self.dataloader[name + '_loader'].get_iterator()):
testx = torch.Tensor(x).to(self.device)
testx = testx.transpose(1, 3)
testy = torch.Tensor(y).to(self.device)
testy = testy.transpose(1, 3)[:, :, :, :self.seq_out_len]
realy.append(testy[:, self.params['out_level'], :, :].squeeze())
with torch.no_grad():
preds = self.model(testx, dummy)
preds = preds.transpose(1, 3)
outputs.append(preds.squeeze())
yhat = torch.cat(outputs, dim=0)
realy = torch.cat(realy, dim=0)
pred = self.scaler.inverse_transform(yhat, self.params['out_level'])
realy = self.scaler.inverse_transform(realy, self.params['out_level'])
mae, mape, rmse = metrics.metric(pred, realy)
return mae, mape, rmse
def test(self, setting, logger):
test_data, test_loader = self._get_data(flag='test')
self.model.eval()
preds = []
trues = []
for i, test_d in enumerate(test_loader):
pred, true = self._process_one_batch(test_data, test_d)
preds.append(pred.detach().cpu().numpy())
trues.append(true.detach().cpu().numpy())
preds = np.array(preds)
trues = np.array(trues)
logger.info('test shape: {} {}'.format(preds.shape, trues.shape))
preds = preds.reshape((-1, preds.shape[-2], preds.shape[-1]))
trues = trues.reshape((-1, trues.shape[-2], trues.shape[-1]))
logger.info('test shape: {} {}'.format(preds.shape, trues.shape))
# result save
folder_path = './results/' + setting + '/'
if not os.path.exists(folder_path):
os.makedirs(folder_path)
mae, mse, rmse, mape, mspe = metric(preds, trues)
logger.info('R{} mse:{}, mae:{}'.format(self.args.rank, mse, mae))
np.save(folder_path + 'metrics.npy',
np.array([mae, mse, rmse, mape, mspe]))
np.save(folder_path + 'pred.npy', preds)
np.save(folder_path + 'true.npy', trues)
return
def predict(self,
query,
threshold=config.default_threshold,
N=config.num_recommendations,
Trigram = False):
start_time = time.time()
query_norm = self.normalizer.convert(query, divNGram=False)
query_vec = self.vectorizer.transform([query_norm])
dist, idx = self.knn.kneighbors(query_vec)
idx = idx[0]
timing_search = "%.2f ms" % ((time.time() - start_time) * 1000)
if len(idx) == 0:
res_json = []
else:
res_json = []
for id in idx:
item = metric(query_norm, self.labels[id], self.normalizer, Trigram=Trigram)
if float(item['lev']) >= threshold:
res_json += [item]
# ====== RE-RANKING =========================================================
res_json = sorted(res_json, key=lambda i: i['lev'], reverse=True)
timing = "%.2f ms" % ((time.time() - start_time) * 1000)
return {'query': query, 'data': res_json, 'time': timing, 'max': N, 'time_search': timing_search,
'threshold': threshold, 'algoritm': self.entryname}
def test(self, setting):
try:
self.test_data = self.test_data
except:
self.test_data, self.test_loader = self._get_data(flag='test',
data_type=100)
preds = []
trues = []
hiss = []
for i, (batch_x, batch_y) in enumerate(self.test_loader):
batch_x = batch_x.double() # .to(self.device)
batch_y = batch_y.double()
outputs = self.model(batch_x).view(-1, 24).detach()
batch_y = batch_y[:, -self.args.pred_len:, -1] # .to(self.device)
pred = outputs.detach().cpu().numpy() # .squeeze()
true = batch_y.detach().cpu().numpy() # .squeeze()
hiss.append(batch_x.detach().cpu().numpy())
preds.append(pred)
trues.append(true)
preds = np.array(preds)
trues = np.array(trues)
print('test shape:', preds.shape, trues.shape)
preds = preds.reshape(-1, 24)
trues = trues.reshape(-1, 24)
data = pd.DataFrame()
for i in range(preds.shape[0]):
data['pred_' + str(i)] = preds[i]
data['true_' + str(i)] = trues[i]
# data.to_csv('result.csv')
print('tset_typeest shape:', preds.shape, trues.shape)
try:
mae, mse, rmse, mape, mspe = metric(preds[:, :, -1], trues[:, :,
-1])
score = evaluate_metrics(preds[:, :, -1], trues[:, :, -1])
except:
mae, mse, rmse, mape, mspe = metric(preds, trues)
score = evaluate_metrics(preds, trues)
print('mse:{}, mae:{}, score:{}'.format(mse, mae, score))
return mse, mae, score
def predict(self, setting, load=False):
"""Prediction Function.
Args:
setting: Name used to be used for prediction
load: whether to load best model
Returns:
mae: Mean absolute error
mse: Mean squared error
rmse: Root mean squared error
mape: Mean absolute percentage error
mspe: Mean squared percentage error
"""
# Create prediction dataset
pred_loader = self._get_data(flag='pred')
# Load best model saved in the checkpoint folder
if load:
path = os.path.join(self.args.checkpoints, setting)
best_model_path = path + '/' + 'checkpoint.pth'
self.model.load_state_dict(torch.load(best_model_path))
# Get model predictions
self.model.eval()
for i, (batch_x, batch_y) in enumerate(pred_loader):
pred, true = self._process_one_batch(batch_x,
batch_y,
validation=True)
if i == 0:
preds = pred.detach().cpu().numpy()
trues = true.detach().cpu().numpy()
else:
preds = np.concatenate((preds, pred.detach().cpu().numpy()),
axis=0)
trues = np.concatenate((trues, true.detach().cpu().numpy()),
axis=0)
if self.args.scale:
# Transform dataset back to orignal form
preds = self.label_scaler.inverse_transform(preds.reshape(-1, 1))
trues = self.label_scaler.inverse_transform(trues.reshape(-1, 1))
# save predictions made by model
folder_path = './results/' + setting + '/'
check_folder = os.path.isdir(folder_path)
if not check_folder:
os.makedirs(folder_path)
np.save(folder_path + 'real_prediction.npy', preds)
# Evaluate the model preformance
mae, mse, rmse, mape, mspe = metric(preds, trues)
print('mse:{}, mae:{}, rmse:{}'.format(mse, mae, rmse))
return mae, mse, rmse, mape, mspe, 0, 0
def predict(self, setting, load=False):
pred_data, pred_loader = self._get_data(flag='pred')
if load:
# checkpoint default "checkpoints"
path = os.path.join(self.args.checkpoints, setting)
best_model_path = path+'/'+'checkpoint.pth'
self.model.load_state_dict(torch.load(best_model_path))
self.model.eval()
# Batch normは、学習の際はバッチ間の平均や分散を計算しています。
# 推論するときは、平均/分散の値が正規化のために使われます。
# まとめると、eval()はdropoutやbatch normの on/offの切替です。
preds = []
trues = []
# i = 0 しか実行しない bach = 1??
for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(pred_loader):
# どこの予測??
pred, true = self._process_one_batch(
pred_data, batch_x, batch_y, batch_x_mark, batch_y_mark)
preds.append(pred.detach().cpu().numpy())
trues.append(true.detach().cpu().numpy())
np.save("./results/attention/attention_target.npy",
batch_x)
# print("Shape of pred on prediction:{}".format(true.shape))
# print("Shape of true on prediction:{}".format(true.shape))
preds = np.array(preds)
preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
trues = np.array(trues)
trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1])
mae, mse, rmse, mape, mspe = metric(preds, trues)
print('mse:{}, mae:{}'.format(mse, mae))
# result save
folder_path = './results/' + setting + '/'
if not os.path.exists(folder_path):
os.makedirs(folder_path)
np.save(folder_path+'metrics.npy',
np.array([mae, mse, rmse, mape, mspe]))
np.save(folder_path+'pred_preds.npy', preds)
np.save(folder_path+'pred_trues.npy', trues)
send_line_notify(message='Predict : mse:{}, mae:{}'.format(mse, mae))
return
def predict(id):
query = Y[id]
query_norm = normalizer.convert(query, divNGram=False)
row = vectorizer.transform([query_norm])
dist, idx = knn.kneighbors(row)
idx = idx[0]
for id in idx:
res = metric(query_norm, Y[id], normalizer, Trigram=False)
if float(res['lev']) >= 0.3:
return float(res['lev'])
def eval(self, src, tgt):
self.model.eval()
src_x = {'v': src}
tgt_x = {'v': tgt}
dummy = torch.zeros(10).requires_grad_()
predict = self.model(src_x, tgt_x, dummy)
real = torch.unsqueeze(tgt[:, 0, :, :], dim=1)
real = self.scaler.inverse_transform(real)
predict = self.scaler.inverse_transform(predict)
mae, mape, rmse = metrics.metric(predict, real)
return mae, mape, rmse
def test(self, setting):
test_data, test_loader = self._get_data(flag='test')
self.model.eval()
preds = []
trues = []
for i, (batch_x, batch_y, batch_x_mark,
batch_y_mark) in enumerate(test_loader):
batch_x = batch_x.double().to(self.device)
batch_y = batch_y.double()
batch_x_mark = batch_x_mark.double().to(self.device)
batch_y_mark = batch_y_mark.double().to(self.device)
# decoder input
dec_inp = torch.zeros_like(
batch_y[:, -self.args.pred_len:, :]).double()
dec_inp = torch.cat([batch_y[:, :self.args.label_len, :], dec_inp],
dim=1).double().to(self.device)
# encoder - decoder
outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)
batch_y = batch_y[:, -self.args.pred_len:, :].to(self.device)
pred = outputs.detach().cpu().numpy() #.squeeze()
true = batch_y.detach().cpu().numpy() #.squeeze()
preds.append(pred)
trues.append(true)
preds = np.array(preds)
trues = np.array(trues)
print('test shape:', preds.shape, trues.shape)
preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1])
print('test shape:', preds.shape, trues.shape)
# result save
folder_path = './results/' + setting + '/'
if not os.path.exists(folder_path):
os.makedirs(folder_path)
mae, mse, rmse, mape, mspe = metric(preds, trues)
print('mse:{}, mae:{}'.format(mse, mae))
np.save(folder_path + 'metrics.npy',
np.array([mae, mse, rmse, mape, mspe]))
np.save(folder_path + 'pred.npy', preds)
np.save(folder_path + 'true.npy', trues)
return
def predict(self,
query,
threshold=config.default_threshold,
N=config.num_recommendations,
Trigram = False):
if self.model == None:
raise Exception("Model is not loaded!")
query = cleanhtml(query)
if self.type != 'trigram':
Trigram = False
query_norm = self.normalizer.convert(query,False)
tokens = self.normalizer.convert(query)
else:
query, query_norm = self.normalizer.get_last_trigram(query)
if query_norm == None:
return {'query': query,
'data': [],
'time': '0 ms',
'max': N,
'time_search': '0 ms',
'threshold': threshold}
else:
Trigram = True
tokens = self.normalizer.generate_ngrams_char(query_norm)
start_time = time.time()
m = MinHash(num_perm=self.permutation)
for s in tokens:
m.update(s.encode('utf8'))
# m e' la query sotto forma di bucket ed N e' il numero max di elementi richiesti
idx_array = np.array(self.model.query(m, N))
timing_search = "%.2f ms" % ((time.time() - start_time) * 1000)
if len(idx_array) == 0:
res_json = []
else:
res_json = []
for doc_retrival in idx_array:
item = metrics.metric(query_norm, doc_retrival, self.normalizer,Trigram=Trigram)
if float(item['lev']) >= threshold:
res_json += [item]
# ====== RE-RANKING =========================================================
res_json = sorted(res_json, key=lambda i: i['lev'], reverse=True)
timing = "%.2f ms" % ((time.time() - start_time) * 1000)
return {'query': query, 'data': res_json, 'time': timing, 'max':N, 'time_search':timing_search, 'threshold':threshold, 'algoritm':self.entryname}
def test(self, setting):
folder_path = './results/' + setting + '/'
if not os.path.exists(folder_path):
os.makedirs(folder_path)
test_data, test_loader = self._get_data(flag='test')
self.model.eval()
preds = []
trues = []
for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(tqdm(test_loader)):
# 5回だけ実行
if i >= 5:
break
pred, true = self._process_one_batch(
test_data, batch_x, batch_y, batch_x_mark, batch_y_mark)
preds.append(pred.detach().cpu().numpy())
trues.append(true.detach().cpu().numpy())
decorder_inputs = batch_x
np.save('./results/test_input/batch_x.npy', batch_x)
np.save('./results/test_input/batch_y.npy', batch_y)
preds = np.array(preds)
trues = np.array(trues)
# print('test shape:', preds.shape, trues.shape)
preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1])
# print('test shape:', preds.shsape, trues.shape)
# result save
mae, mse, rmse, mape, mspe = metric(preds, trues)
print('mse:{}, mae:{}'.format(mse, mae))
np.save(folder_path+'metrics.npy',
np.array([mae, mse, rmse, mape, mspe]))
np.save(folder_path+'pred.npy', preds)
np.save(folder_path+'true.npy', trues)
send_line_notify(
message='test : mse:{}, mae:{}'.format(mse, mae)+"\nsetting : "+setting)
return
def ev_valid(self,name):
self.model.eval()
y_preds = []
y_truths = []
for iter, (x, y) in enumerate(self.dataloader[name+'_loader'].get_iterator()):
x, y = self._prepare_data(x, y)
with torch.no_grad():
preds = self.model(x,y)
y_preds.append(preds)
y = y[..., self.params['out_level']].squeeze()
y_truths.append(y)
y_preds = torch.cat(y_preds, axis=1)
y_truths = torch.cat(y_truths, axis=1)
y_preds = self.scaler.inverse_transform(y_preds,self.params['out_level'])
y_truths = self.scaler.inverse_transform(y_truths,self.params['out_level'])
mae, mape, rmse = metrics.metric(y_preds, y_truths)
return mae, mape, rmse
def ev_test(self, name):
self.model.eval()
outputs = []
realy = []
# realy = torch.Tensor(self.dataloader['y_'+name]).to(self.device)
# realy = realy.transpose(1, 3)[:, 0, :, :self.seq_out_len]
dummy = torch.zeros(10).requires_grad_()
for iter, (x, y) in enumerate(
self.dataloader[name + '_loader'].get_iterator()):
testx = torch.Tensor(x).to(self.device)
testx = testx.transpose(1, 3)
src_x = {'v': testx}
testy = torch.Tensor(y).to(self.device)
testy = testy.transpose(1, 3)[:, :, :, :self.seq_out_len]
realy.append(testy[:, 0, :, :].squeeze())
tgt_x = {'v': testy}
with torch.no_grad():
preds = self.model(src_x, tgt_x, dummy)
outputs.append(preds.squeeze())
yhat = torch.cat(outputs, dim=0)
realy = torch.cat(realy, dim=0)
mae = []
mape = []
rmse = []
for i in range(self.seq_out_len):
pred = self.scaler.inverse_transform(yhat[:, :, i])
real = realy[:, :, i]
real = self.scaler.inverse_transform(real)
results = metrics.metric(pred, real)
log = 'Evaluate best model on ' + name + ' data for horizon {:d}, MAE: {:.4f}, MAPE: {:.4f}, RMSE: {:.4f}'
print(log.format(i + 1, results[0], results[1], results[2]))
mae.append(results[0])
mape.append(results[1])
rmse.append(results[2])
return mae, mape, rmse
def test(self, setting_identifier):
test_data, test_loader = self._get_data(flag='test')
self.model.eval()
preds = []
trues = []
for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(test_loader):
pred, true = self._process_one_batch(
test_data, batch_x, batch_y, batch_x_mark, batch_y_mark)
preds.append(pred.detach().cpu().numpy())
trues.append(true.detach().cpu().numpy())
preds = np.array(preds)
trues = np.array(trues)
if self.args.test_recover:
preds = self.data_set.inverse_transform(preds)
trues = self.data_set.inverse_transform(trues)
print('original test shape:', preds.shape, trues.shape)
preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1])
print('reshaped test shape:', preds.shape, trues.shape)
# result save
folder_path = './results/' + setting_identifier +'/'
if not os.path.exists(folder_path):
os.makedirs(folder_path)
mae, mse, rmse, mape, mspe = metric(preds, trues)
print('mse:{}, mae:{}'.format(mse, mae))
np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe]))
np.save(folder_path+'pred.npy', preds)
np.save(folder_path+'true.npy', trues)
return
def test(self, setting):
try:
self.test_data = self.test_data
except:
self.test_data, self.test_loader = self._get_data(flag='test',
data_type=100)
# path = './checkpoints/' + setting + '/' + 'checkpoint.pth'
# try:
# self.model.load_state_dict(torch.load(path))
# except:
# print('Model can not be load from', path)
# self.model.eval()
preds = []
trues = []
hiss = []
for i, (batch_x, batch_y, batch_x_mark,
batch_y_mark) in enumerate(self.test_loader):
batch_x = batch_x.double() # .to(self.device)
batch_y = batch_y.double()
batch_x_mark = batch_x_mark.double() # .to(self.device)
batch_y_mark = batch_y_mark.double() # .to(self.device)
# decoder input
dec_inp = torch.zeros_like(
batch_y[:, -self.args.pred_len:, :]).double()
dec_inp = torch.cat([batch_y[:, :self.args.label_len, :], dec_inp],
dim=1).double() # .to(self.device)
# encoder - decoder
outputs = self.model(batch_x, batch_x_mark, dec_inp,
batch_y_mark).view(-1, 24).detach()
batch_y = batch_y[:, -self.args.pred_len:, -1] # .to(self.device)
pred = outputs.detach().cpu().numpy() # .squeeze()
true = batch_y.detach().cpu().numpy() # .squeeze()
hiss.append(batch_x.detach().cpu().numpy())
preds.append(pred)
trues.append(true)
preds = np.array(preds)
trues = np.array(trues)
print('test shape:', preds.shape, trues.shape)
preds = preds.reshape(-1, 24)
trues = trues.reshape(-1, 24)
data = pd.DataFrame()
for i in range(preds.shape[0]):
data['pred_' + str(i)] = preds[i]
data['true_' + str(i)] = trues[i]
data.to_csv('result.csv')
print('test shape:', preds.shape, trues.shape)
try:
mae, mse, rmse, mape, mspe = metric(preds[:, :, -1], trues[:, :,
-1])
score = evaluate_metrics(preds[:, :, -1], trues[:, :, -1])
except:
mae, mse, rmse, mape, mspe = metric(preds, trues)
score = evaluate_metrics(preds, trues)
print('mse:{}, mae:{}, score:{}'.format(mse, mae, score))
return mse, mae, score
return
def anderson_iteration(X, U, V, labels, p, logger):
import time
t = 0
mmax = p.mmax or 3
gamma = p.gamma
epsilon = p.epsilon
max_iteration = p.max_iterations or 300
V_old = update_V(V, U, X, epsilon)
U_new = U
old_E = np.Infinity
VAUt = None
aa_ndim = reduce(operator.mul, V.shape)
aa_shape = (1, aa_ndim)
accelerator = None
# energy = Energy()
while True:
start = time.time()
U_now = solve_U(X, V_old, gamma, epsilon)
end = time.time()
print('solve u', end - start)
_, converged = U_converged(U_now, U_new)
print(_)
U_new = U_now
if converged:
return U_new, V_old, t, metric(U_new, labels)
# delta_U, is_converged = U_converged(U_new, U_old)
new_E = E(U_new, V_old, X, gamma, epsilon)
if new_E >= old_E:
V_old = VAUt
U_new = solve_U(X, V_old, gamma, epsilon)
new_E = E(U_new, V_old, X, gamma, epsilon)
accelerator.replace(V_old.reshape(aa_shape))
# if energy.converged(new_E):
# return U_new, V_old, t, metric(U_new, labels)
if t > max_iteration:
return U_new, V_old, t, metric(U_new, labels)
# energy.add(new_E)
logger.log_middle(new_E, metric(U_new, labels))
start = time.time()
VAUt = update_V(V_old, U_new, X, epsilon)
end = time.time()
print('update v', end - start)
if t == 0:
accelerator = Anderson(mmax, aa_ndim, VAUt.reshape(aa_shape))
V_new = VAUt
else:
accelerator.set_G(VAUt.reshape(aa_shape))
V_new = accelerator.compute().reshape(V.shape)
V_old = V_new
# U_old = U_new
old_E = new_E
t += 1
def predict(self, setting, load=False):
"""Prediction Function.
Args:
setting: Name used to be used for prediction
load: whether to load best model
Returns:
mae: Mean absolute error
mse: Mean squared error
rmse: Root mean squared error
mape: Mean absolute percentage error
mspe: Mean squared percentage error
"""
# Create prediction dataset
pred_loader = self._get_data(flag="pred")
# Load best model saved in the checkpoint folder
if load:
path = os.path.join(self.args.checkpoints, setting)
best_model_path = path + "/" + "checkpoint.pth"
self.model.load_state_dict(torch.load(best_model_path))
self.error_scaler = pickle.load(open(path + "/" + "std_scaler.bin", "rb"))
self.model.eval()
for i, (batch_x, index, batch_y) in enumerate(pred_loader):
# get the error made by experts in each previous time step
past_val_errors = self.get_past_errors(index, "test")
# get the output of the model (Argmax weights * expert),
# each expert output and weights assigned to each expert
output, expert_output, weights, true = self._process_one_batch(
batch_x, batch_y, past_errors=past_val_errors, validation=True)
# the choosen is the expert with higest weight by gate network
_, choosen = torch.max(weights, 1)
# get the error made by each expert
error = self.model_assignment_error(expert_output, true)
# update the error matrix with new errors note that we update with
# only first time step error to prevent information leakage
error = torch.Tensor(
self.error_scaler.transform(
error.detach().cpu().numpy().flatten().reshape(-1, 1))).reshape(
-1, self.num_experts).to(self.device)
self.set_past_errors(error.detach(), index, "test")
if i == 0:
preds = output.detach().cpu().numpy()
trues = true.detach().cpu().numpy()
assignment = choosen.detach().cpu().numpy()
else:
preds = np.concatenate((preds, output.detach().cpu().numpy()), axis=0)
trues = np.concatenate((trues, true.detach().cpu().numpy()), axis=0)
assignment = np.concatenate(
(assignment, choosen.detach().cpu().numpy()), axis=0)
if self.args.plot:
assignment = np.array(assignment).squeeze()
plt.figure()
plt.plot(assignment)
plt.title("assignment over time")
plt.legend()
plt.savefig("./Graphs/" + setting + "_Final_assignment.png")
plt.show()
plt.close()
if self.args.scale:
# Transform dataset back to orignal form
preds = self.label_scaler.inverse_transform(preds.reshape(-1, 1))
trues = self.label_scaler.inverse_transform(trues.reshape(-1, 1))
# save predictions made by model
folder_path = "./results/" + setting + "/"
check_folder = os.path.isdir(folder_path)
if not check_folder:
os.makedirs(folder_path)
np.save(folder_path + "real_prediction.npy", preds)
# Evaluate the model performance
mae, mse, rmse, mape, mspe = metric(preds, trues)
print("mse:{}, mae:{}, rmse:{}".format(mse, mae, rmse))
self.past_test_error = torch.zeros(
(len(self.data.test_x), self.num_experts),
requires_grad=False).to(self.device)
_, upper_bound, oracle_acc = self.get_upper_bound_accuracy(
pred_loader, flag="test", plot=self.args.plot, setting=setting)
return mae, mse, rmse, mape, mspe, upper_bound, oracle_acc
def get_upper_bound_accuracy(self,
test_loader,
hide_print=False,
flag="train",
plot=False,
setting=None):
"""Function that calculates upper bound, oracle accuracy and mse.
Args:
test_loader: data loader
hide_print: a boolean to indicate whether to print each expert accuracy
flag: a string indicating dataset type
plot: a boolean to indicate whether plot oracle assignment over time
and expert error
setting: a string containing model name
Returns:
mse: mean squared error for entire IME
upper_bound: best possible mse had the gate network choosen the best
expert
oracle_acc: Oracle accuracy showing the percentage of times the gate
network chooses the best expert
"""
self.model.eval()
upper_bound = 0
expert = [0 for i in range(self.num_experts)]
expert_error = [[] for i in range(self.num_experts)]
for i, (batch_x, index, batch_y) in enumerate(test_loader):
# get the error made by experts in each previous time step
past_val_errors = self.get_past_errors(index, flag)
# get the output of the model (Argmax weights * expert),
# each expert output and weights assigned to each expert
output, expert_output, weights, true = self._process_one_batch(
batch_x, batch_y, past_errors=past_val_errors, validation=True)
# get the error made by each expert
error = self.model_assignment_error(expert_output, true)
# update the error matrix with new errors note that we update with
# only first time step error to prevent information leakage
self.set_past_errors(error.detach(), index, flag)
# the best accuracy would be the one that gives minimum error.
_, actual_best = torch.min(error, 1)
# the choosen is the expert with higest weight by gate network
_, choosen = torch.max(weights, 1)
upper_bound += torch.mean(torch.min(error, 1)[0])
for y in range(self.num_experts):
expert[y] += torch.mean(error[:, y]).item()
expert_error[y].append(torch.mean(error[:, y]).item())
# or the entire dataset get predictions and labels
if i == 0:
preds = output.detach().cpu().numpy()
trues = true.detach().cpu().numpy()
else:
preds = np.concatenate((preds, output.detach().cpu().numpy()), axis=0)
trues = np.concatenate((trues, true.detach().cpu().numpy()), axis=0)
# for the entire dataset get best expert (with minimum error)
# and choosen expert by gate network
if i == 0:
actual_best_expert = actual_best.detach().cpu().numpy()
choosen_expert = choosen.squeeze(-1).detach().cpu().numpy()
else:
actual_best_expert = np.concatenate(
(actual_best_expert, actual_best.detach().cpu().numpy()), axis=0)
choosen_expert = np.concatenate(
(choosen_expert, choosen.squeeze(-1).detach().cpu().numpy()),
axis=0)
# Oracle accuracy calcuates the percentage of times the gate network
# chooses the best expert
oracle_acc = accuracy_score(actual_best_expert, choosen_expert)
upper_bound = upper_bound.item() / len(test_loader)
# update past error matrix
if flag == "test":
self.past_test_error = torch.Tensor(
self.error_scaler.transform(
self.past_test_error.detach().cpu().numpy().flatten().reshape(
-1, 1))).reshape(-1, self.num_experts).to(self.device)
elif flag == "train":
self.error_scaler.fit(
self.past_train_error.detach().cpu().numpy().flatten().reshape(-1, 1))
self.past_train_error = torch.Tensor(
self.error_scaler.transform(
self.past_train_error.detach().cpu().numpy().flatten().reshape(
-1, 1))).reshape(-1, self.num_experts).to(self.device)
else:
self.past_val_error = torch.Tensor(
self.error_scaler.transform(
self.past_val_error.detach().cpu().numpy().flatten().reshape(
-1, 1))).reshape(-1, self.num_experts).to(self.device)
# get accuracy metrics
_, mse, _, _, _ = metric(preds, trues)
if not hide_print:
print(flag)
print("oracle accuracy: {0:.7f}".format(oracle_acc))
print("Upper bound: {0:.7f}".format(upper_bound))
for y in range(self.num_experts):
print("Expert {0}: {1:.7f}".format(y, expert[y] / len(test_loader)))
print("MSE: {0:.7f}".format(mse))
print()
if plot:
plt.figure()
plt.plot(actual_best_expert)
plt.title("oracle assignment")
plt.legend()
plt.savefig("./Graphs/" + setting + "_oracle_assignment_" + flag + ".png")
plt.show()
plt.close()
plt.figure()
_, axs = plt.subplots(self.num_experts, 1)
for y in range(self.num_experts):
axs[y].plot(expert_error[y])
axs[y].set_xlabel("expert" + str(y))
plt.savefig("./Graphs/" + setting + "_expert_error_" + flag + ".png")
plt.show()
plt.close()
return mse, upper_bound, oracle_acc
