def ev_test(self, name): self.model.eval() y_preds = [] y_truths = [] for iter, (x, y) in enumerate(self.dataloader[name+'_loader'].get_iterator()): x, y = self._prepare_data(x, y) with torch.no_grad(): preds = self.model(x,y) y_preds.append(preds) y = y[..., self.params['out_level']].squeeze() y_truths.append(y) y_preds = torch.cat(y_preds, axis=1) y_truths = torch.cat(y_truths, axis=1) mae = [] mape = [] rmse = [] for i in range(self.seq_out_len): pred = self.scaler.inverse_transform(y_preds[i,...],self.params['out_level']) real = self.scaler.inverse_transform(y_truths[i,...],self.params['out_level']) results = metrics.metric(pred, real) log = 'Evaluate best model on test data for horizon {:d}, Test MAE: {:.4f}, Test MAPE: {:.4f}, Test RMSE: {:.4f}' print(log.format(i + 1, results[0], results[1], results[2])) mae.append(results[0]) mape.append(results[1]) rmse.append(results[2]) return mae, mape, rmse
def test(self, setting): test_data, test_loader = self._get_data(flag='test') self.model.eval() preds = [] trues = [] for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(test_loader): batch_x = batch_x.float().to(self.device) batch_y = batch_y.float() batch_x_mark = batch_x_mark.float().to(self.device) batch_y_mark = batch_y_mark.float().to(self.device) # decoder input dec_inp = torch.zeros_like(batch_y[:,-self.args.pred_len:,:]).float() dec_inp = torch.cat([batch_y[:,:self.args.label_len,:], dec_inp], dim=1).float().to(self.device) # encoder - decoder if self.args.use_amp: with torch.cuda.amp.autocast(): if self.args.output_attention: outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0] else: outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) else: if self.args.output_attention: outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0] else: outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) if self.args.inverse: outputs = test_data.inverse_transform(outputs) f_dim = -1 if self.args.features=='MS' else 0 batch_y = batch_y[:,-self.args.pred_len:,f_dim:].to(self.device) pred = outputs.detach().cpu().numpy()#.squeeze() true = batch_y.detach().cpu().numpy()#.squeeze() preds.append(pred) trues.append(true) preds = np.array(preds) trues = np.array(trues) print('test shape:', preds.shape, trues.shape) preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) print('test shape:', preds.shape, trues.shape) # result save folder_path = self.args.results_path + setting +'/' if not os.path.exists(folder_path): os.makedirs(folder_path) mae, mse, rmse, mape, mspe = metric(preds, trues) print('mse:{}, mae:{}'.format(mse, mae)) np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) np.save(folder_path+'pred.npy', preds) np.save(folder_path+'true.npy', trues) return
def ev_valid(self, name): self.model.eval() outputs = [] realy = [] dummy = torch.zeros(10).requires_grad_() for iter, (x, y) in enumerate( self.dataloader[name + '_loader'].get_iterator()): testx = torch.Tensor(x).to(self.device) testx = testx.transpose(1, 3) testy = torch.Tensor(y).to(self.device) testy = testy.transpose(1, 3)[:, :, :, :self.seq_out_len] realy.append(testy[:, self.params['out_level'], :, :].squeeze()) with torch.no_grad(): preds = self.model(testx, dummy) preds = preds.transpose(1, 3) outputs.append(preds.squeeze()) yhat = torch.cat(outputs, dim=0) realy = torch.cat(realy, dim=0) pred = self.scaler.inverse_transform(yhat, self.params['out_level']) realy = self.scaler.inverse_transform(realy, self.params['out_level']) mae, mape, rmse = metrics.metric(pred, realy) return mae, mape, rmse
def test(self, setting, logger): test_data, test_loader = self._get_data(flag='test') self.model.eval() preds = [] trues = [] for i, test_d in enumerate(test_loader): pred, true = self._process_one_batch(test_data, test_d) preds.append(pred.detach().cpu().numpy()) trues.append(true.detach().cpu().numpy()) preds = np.array(preds) trues = np.array(trues) logger.info('test shape: {} {}'.format(preds.shape, trues.shape)) preds = preds.reshape((-1, preds.shape[-2], preds.shape[-1])) trues = trues.reshape((-1, trues.shape[-2], trues.shape[-1])) logger.info('test shape: {} {}'.format(preds.shape, trues.shape)) # result save folder_path = './results/' + setting + '/' if not os.path.exists(folder_path): os.makedirs(folder_path) mae, mse, rmse, mape, mspe = metric(preds, trues) logger.info('R{} mse:{}, mae:{}'.format(self.args.rank, mse, mae)) np.save(folder_path + 'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) np.save(folder_path + 'pred.npy', preds) np.save(folder_path + 'true.npy', trues) return
def predict(self, query, threshold=config.default_threshold, N=config.num_recommendations, Trigram = False): start_time = time.time() query_norm = self.normalizer.convert(query, divNGram=False) query_vec = self.vectorizer.transform([query_norm]) dist, idx = self.knn.kneighbors(query_vec) idx = idx[0] timing_search = "%.2f ms" % ((time.time() - start_time) * 1000) if len(idx) == 0: res_json = [] else: res_json = [] for id in idx: item = metric(query_norm, self.labels[id], self.normalizer, Trigram=Trigram) if float(item['lev']) >= threshold: res_json += [item] # ====== RE-RANKING ========================================================= res_json = sorted(res_json, key=lambda i: i['lev'], reverse=True) timing = "%.2f ms" % ((time.time() - start_time) * 1000) return {'query': query, 'data': res_json, 'time': timing, 'max': N, 'time_search': timing_search, 'threshold': threshold, 'algoritm': self.entryname}
def test(self, setting): try: self.test_data = self.test_data except: self.test_data, self.test_loader = self._get_data(flag='test', data_type=100) preds = [] trues = [] hiss = [] for i, (batch_x, batch_y) in enumerate(self.test_loader): batch_x = batch_x.double() # .to(self.device) batch_y = batch_y.double() outputs = self.model(batch_x).view(-1, 24).detach() batch_y = batch_y[:, -self.args.pred_len:, -1] # .to(self.device) pred = outputs.detach().cpu().numpy() # .squeeze() true = batch_y.detach().cpu().numpy() # .squeeze() hiss.append(batch_x.detach().cpu().numpy()) preds.append(pred) trues.append(true) preds = np.array(preds) trues = np.array(trues) print('test shape:', preds.shape, trues.shape) preds = preds.reshape(-1, 24) trues = trues.reshape(-1, 24) data = pd.DataFrame() for i in range(preds.shape[0]): data['pred_' + str(i)] = preds[i] data['true_' + str(i)] = trues[i] # data.to_csv('result.csv') print('tset_typeest shape:', preds.shape, trues.shape) try: mae, mse, rmse, mape, mspe = metric(preds[:, :, -1], trues[:, :, -1]) score = evaluate_metrics(preds[:, :, -1], trues[:, :, -1]) except: mae, mse, rmse, mape, mspe = metric(preds, trues) score = evaluate_metrics(preds, trues) print('mse:{}, mae:{}, score:{}'.format(mse, mae, score)) return mse, mae, score
def predict(self, setting, load=False): """Prediction Function. Args: setting: Name used to be used for prediction load: whether to load best model Returns: mae: Mean absolute error mse: Mean squared error rmse: Root mean squared error mape: Mean absolute percentage error mspe: Mean squared percentage error """ # Create prediction dataset pred_loader = self._get_data(flag='pred') # Load best model saved in the checkpoint folder if load: path = os.path.join(self.args.checkpoints, setting) best_model_path = path + '/' + 'checkpoint.pth' self.model.load_state_dict(torch.load(best_model_path)) # Get model predictions self.model.eval() for i, (batch_x, batch_y) in enumerate(pred_loader): pred, true = self._process_one_batch(batch_x, batch_y, validation=True) if i == 0: preds = pred.detach().cpu().numpy() trues = true.detach().cpu().numpy() else: preds = np.concatenate((preds, pred.detach().cpu().numpy()), axis=0) trues = np.concatenate((trues, true.detach().cpu().numpy()), axis=0) if self.args.scale: # Transform dataset back to orignal form preds = self.label_scaler.inverse_transform(preds.reshape(-1, 1)) trues = self.label_scaler.inverse_transform(trues.reshape(-1, 1)) # save predictions made by model folder_path = './results/' + setting + '/' check_folder = os.path.isdir(folder_path) if not check_folder: os.makedirs(folder_path) np.save(folder_path + 'real_prediction.npy', preds) # Evaluate the model preformance mae, mse, rmse, mape, mspe = metric(preds, trues) print('mse:{}, mae:{}, rmse:{}'.format(mse, mae, rmse)) return mae, mse, rmse, mape, mspe, 0, 0
def predict(self, setting, load=False): pred_data, pred_loader = self._get_data(flag='pred') if load: # checkpoint default "checkpoints" path = os.path.join(self.args.checkpoints, setting) best_model_path = path+'/'+'checkpoint.pth' self.model.load_state_dict(torch.load(best_model_path)) self.model.eval() # Batch normは、学習の際はバッチ間の平均や分散を計算しています。 # 推論するときは、平均/分散の値が正規化のために使われます。 # まとめると、eval()はdropoutやbatch normの on/offの切替です。 preds = [] trues = [] # i = 0 しか実行しない bach = 1?? for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(pred_loader): # どこの予測?? pred, true = self._process_one_batch( pred_data, batch_x, batch_y, batch_x_mark, batch_y_mark) preds.append(pred.detach().cpu().numpy()) trues.append(true.detach().cpu().numpy()) np.save("./results/attention/attention_target.npy", batch_x) # print("Shape of pred on prediction:{}".format(true.shape)) # print("Shape of true on prediction:{}".format(true.shape)) preds = np.array(preds) preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) trues = np.array(trues) trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) mae, mse, rmse, mape, mspe = metric(preds, trues) print('mse:{}, mae:{}'.format(mse, mae)) # result save folder_path = './results/' + setting + '/' if not os.path.exists(folder_path): os.makedirs(folder_path) np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) np.save(folder_path+'pred_preds.npy', preds) np.save(folder_path+'pred_trues.npy', trues) send_line_notify(message='Predict : mse:{}, mae:{}'.format(mse, mae)) return
def predict(id): query = Y[id] query_norm = normalizer.convert(query, divNGram=False) row = vectorizer.transform([query_norm]) dist, idx = knn.kneighbors(row) idx = idx[0] for id in idx: res = metric(query_norm, Y[id], normalizer, Trigram=False) if float(res['lev']) >= 0.3: return float(res['lev'])
def eval(self, src, tgt): self.model.eval() src_x = {'v': src} tgt_x = {'v': tgt} dummy = torch.zeros(10).requires_grad_() predict = self.model(src_x, tgt_x, dummy) real = torch.unsqueeze(tgt[:, 0, :, :], dim=1) real = self.scaler.inverse_transform(real) predict = self.scaler.inverse_transform(predict) mae, mape, rmse = metrics.metric(predict, real) return mae, mape, rmse
def test(self, setting): test_data, test_loader = self._get_data(flag='test') self.model.eval() preds = [] trues = [] for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(test_loader): batch_x = batch_x.double().to(self.device) batch_y = batch_y.double() batch_x_mark = batch_x_mark.double().to(self.device) batch_y_mark = batch_y_mark.double().to(self.device) # decoder input dec_inp = torch.zeros_like( batch_y[:, -self.args.pred_len:, :]).double() dec_inp = torch.cat([batch_y[:, :self.args.label_len, :], dec_inp], dim=1).double().to(self.device) # encoder - decoder outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark) batch_y = batch_y[:, -self.args.pred_len:, :].to(self.device) pred = outputs.detach().cpu().numpy() #.squeeze() true = batch_y.detach().cpu().numpy() #.squeeze() preds.append(pred) trues.append(true) preds = np.array(preds) trues = np.array(trues) print('test shape:', preds.shape, trues.shape) preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) print('test shape:', preds.shape, trues.shape) # result save folder_path = './results/' + setting + '/' if not os.path.exists(folder_path): os.makedirs(folder_path) mae, mse, rmse, mape, mspe = metric(preds, trues) print('mse:{}, mae:{}'.format(mse, mae)) np.save(folder_path + 'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) np.save(folder_path + 'pred.npy', preds) np.save(folder_path + 'true.npy', trues) return
def predict(self, query, threshold=config.default_threshold, N=config.num_recommendations, Trigram = False): if self.model == None: raise Exception("Model is not loaded!") query = cleanhtml(query) if self.type != 'trigram': Trigram = False query_norm = self.normalizer.convert(query,False) tokens = self.normalizer.convert(query) else: query, query_norm = self.normalizer.get_last_trigram(query) if query_norm == None: return {'query': query, 'data': [], 'time': '0 ms', 'max': N, 'time_search': '0 ms', 'threshold': threshold} else: Trigram = True tokens = self.normalizer.generate_ngrams_char(query_norm) start_time = time.time() m = MinHash(num_perm=self.permutation) for s in tokens: m.update(s.encode('utf8')) # m e' la query sotto forma di bucket ed N e' il numero max di elementi richiesti idx_array = np.array(self.model.query(m, N)) timing_search = "%.2f ms" % ((time.time() - start_time) * 1000) if len(idx_array) == 0: res_json = [] else: res_json = [] for doc_retrival in idx_array: item = metrics.metric(query_norm, doc_retrival, self.normalizer,Trigram=Trigram) if float(item['lev']) >= threshold: res_json += [item] # ====== RE-RANKING ========================================================= res_json = sorted(res_json, key=lambda i: i['lev'], reverse=True) timing = "%.2f ms" % ((time.time() - start_time) * 1000) return {'query': query, 'data': res_json, 'time': timing, 'max':N, 'time_search':timing_search, 'threshold':threshold, 'algoritm':self.entryname}
def test(self, setting): folder_path = './results/' + setting + '/' if not os.path.exists(folder_path): os.makedirs(folder_path) test_data, test_loader = self._get_data(flag='test') self.model.eval() preds = [] trues = [] for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(tqdm(test_loader)): # 5回だけ実行 if i >= 5: break pred, true = self._process_one_batch( test_data, batch_x, batch_y, batch_x_mark, batch_y_mark) preds.append(pred.detach().cpu().numpy()) trues.append(true.detach().cpu().numpy()) decorder_inputs = batch_x np.save('./results/test_input/batch_x.npy', batch_x) np.save('./results/test_input/batch_y.npy', batch_y) preds = np.array(preds) trues = np.array(trues) # print('test shape:', preds.shape, trues.shape) preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) # print('test shape:', preds.shsape, trues.shape) # result save mae, mse, rmse, mape, mspe = metric(preds, trues) print('mse:{}, mae:{}'.format(mse, mae)) np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) np.save(folder_path+'pred.npy', preds) np.save(folder_path+'true.npy', trues) send_line_notify( message='test : mse:{}, mae:{}'.format(mse, mae)+"\nsetting : "+setting) return
def ev_valid(self,name): self.model.eval() y_preds = [] y_truths = [] for iter, (x, y) in enumerate(self.dataloader[name+'_loader'].get_iterator()): x, y = self._prepare_data(x, y) with torch.no_grad(): preds = self.model(x,y) y_preds.append(preds) y = y[..., self.params['out_level']].squeeze() y_truths.append(y) y_preds = torch.cat(y_preds, axis=1) y_truths = torch.cat(y_truths, axis=1) y_preds = self.scaler.inverse_transform(y_preds,self.params['out_level']) y_truths = self.scaler.inverse_transform(y_truths,self.params['out_level']) mae, mape, rmse = metrics.metric(y_preds, y_truths) return mae, mape, rmse
def ev_test(self, name): self.model.eval() outputs = [] realy = [] # realy = torch.Tensor(self.dataloader['y_'+name]).to(self.device) # realy = realy.transpose(1, 3)[:, 0, :, :self.seq_out_len] dummy = torch.zeros(10).requires_grad_() for iter, (x, y) in enumerate( self.dataloader[name + '_loader'].get_iterator()): testx = torch.Tensor(x).to(self.device) testx = testx.transpose(1, 3) src_x = {'v': testx} testy = torch.Tensor(y).to(self.device) testy = testy.transpose(1, 3)[:, :, :, :self.seq_out_len] realy.append(testy[:, 0, :, :].squeeze()) tgt_x = {'v': testy} with torch.no_grad(): preds = self.model(src_x, tgt_x, dummy) outputs.append(preds.squeeze()) yhat = torch.cat(outputs, dim=0) realy = torch.cat(realy, dim=0) mae = [] mape = [] rmse = [] for i in range(self.seq_out_len): pred = self.scaler.inverse_transform(yhat[:, :, i]) real = realy[:, :, i] real = self.scaler.inverse_transform(real) results = metrics.metric(pred, real) log = 'Evaluate best model on ' + name + ' data for horizon {:d}, MAE: {:.4f}, MAPE: {:.4f}, RMSE: {:.4f}' print(log.format(i + 1, results[0], results[1], results[2])) mae.append(results[0]) mape.append(results[1]) rmse.append(results[2]) return mae, mape, rmse
def test(self, setting_identifier): test_data, test_loader = self._get_data(flag='test') self.model.eval() preds = [] trues = [] for i, (batch_x,batch_y,batch_x_mark,batch_y_mark) in enumerate(test_loader): pred, true = self._process_one_batch( test_data, batch_x, batch_y, batch_x_mark, batch_y_mark) preds.append(pred.detach().cpu().numpy()) trues.append(true.detach().cpu().numpy()) preds = np.array(preds) trues = np.array(trues) if self.args.test_recover: preds = self.data_set.inverse_transform(preds) trues = self.data_set.inverse_transform(trues) print('original test shape:', preds.shape, trues.shape) preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1]) trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1]) print('reshaped test shape:', preds.shape, trues.shape) # result save folder_path = './results/' + setting_identifier +'/' if not os.path.exists(folder_path): os.makedirs(folder_path) mae, mse, rmse, mape, mspe = metric(preds, trues) print('mse:{}, mae:{}'.format(mse, mae)) np.save(folder_path+'metrics.npy', np.array([mae, mse, rmse, mape, mspe])) np.save(folder_path+'pred.npy', preds) np.save(folder_path+'true.npy', trues) return
def test(self, setting): try: self.test_data = self.test_data except: self.test_data, self.test_loader = self._get_data(flag='test', data_type=100) # path = './checkpoints/' + setting + '/' + 'checkpoint.pth' # try: # self.model.load_state_dict(torch.load(path)) # except: # print('Model can not be load from', path) # self.model.eval() preds = [] trues = [] hiss = [] for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(self.test_loader): batch_x = batch_x.double() # .to(self.device) batch_y = batch_y.double() batch_x_mark = batch_x_mark.double() # .to(self.device) batch_y_mark = batch_y_mark.double() # .to(self.device) # decoder input dec_inp = torch.zeros_like( batch_y[:, -self.args.pred_len:, :]).double() dec_inp = torch.cat([batch_y[:, :self.args.label_len, :], dec_inp], dim=1).double() # .to(self.device) # encoder - decoder outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark).view(-1, 24).detach() batch_y = batch_y[:, -self.args.pred_len:, -1] # .to(self.device) pred = outputs.detach().cpu().numpy() # .squeeze() true = batch_y.detach().cpu().numpy() # .squeeze() hiss.append(batch_x.detach().cpu().numpy()) preds.append(pred) trues.append(true) preds = np.array(preds) trues = np.array(trues) print('test shape:', preds.shape, trues.shape) preds = preds.reshape(-1, 24) trues = trues.reshape(-1, 24) data = pd.DataFrame() for i in range(preds.shape[0]): data['pred_' + str(i)] = preds[i] data['true_' + str(i)] = trues[i] data.to_csv('result.csv') print('test shape:', preds.shape, trues.shape) try: mae, mse, rmse, mape, mspe = metric(preds[:, :, -1], trues[:, :, -1]) score = evaluate_metrics(preds[:, :, -1], trues[:, :, -1]) except: mae, mse, rmse, mape, mspe = metric(preds, trues) score = evaluate_metrics(preds, trues) print('mse:{}, mae:{}, score:{}'.format(mse, mae, score)) return mse, mae, score return
def anderson_iteration(X, U, V, labels, p, logger): import time t = 0 mmax = p.mmax or 3 gamma = p.gamma epsilon = p.epsilon max_iteration = p.max_iterations or 300 V_old = update_V(V, U, X, epsilon) U_new = U old_E = np.Infinity VAUt = None aa_ndim = reduce(operator.mul, V.shape) aa_shape = (1, aa_ndim) accelerator = None # energy = Energy() while True: start = time.time() U_now = solve_U(X, V_old, gamma, epsilon) end = time.time() print('solve u', end - start) _, converged = U_converged(U_now, U_new) print(_) U_new = U_now if converged: return U_new, V_old, t, metric(U_new, labels) # delta_U, is_converged = U_converged(U_new, U_old) new_E = E(U_new, V_old, X, gamma, epsilon) if new_E >= old_E: V_old = VAUt U_new = solve_U(X, V_old, gamma, epsilon) new_E = E(U_new, V_old, X, gamma, epsilon) accelerator.replace(V_old.reshape(aa_shape)) # if energy.converged(new_E): # return U_new, V_old, t, metric(U_new, labels) if t > max_iteration: return U_new, V_old, t, metric(U_new, labels) # energy.add(new_E) logger.log_middle(new_E, metric(U_new, labels)) start = time.time() VAUt = update_V(V_old, U_new, X, epsilon) end = time.time() print('update v', end - start) if t == 0: accelerator = Anderson(mmax, aa_ndim, VAUt.reshape(aa_shape)) V_new = VAUt else: accelerator.set_G(VAUt.reshape(aa_shape)) V_new = accelerator.compute().reshape(V.shape) V_old = V_new # U_old = U_new old_E = new_E t += 1
def predict(self, setting, load=False): """Prediction Function. Args: setting: Name used to be used for prediction load: whether to load best model Returns: mae: Mean absolute error mse: Mean squared error rmse: Root mean squared error mape: Mean absolute percentage error mspe: Mean squared percentage error """ # Create prediction dataset pred_loader = self._get_data(flag="pred") # Load best model saved in the checkpoint folder if load: path = os.path.join(self.args.checkpoints, setting) best_model_path = path + "/" + "checkpoint.pth" self.model.load_state_dict(torch.load(best_model_path)) self.error_scaler = pickle.load(open(path + "/" + "std_scaler.bin", "rb")) self.model.eval() for i, (batch_x, index, batch_y) in enumerate(pred_loader): # get the error made by experts in each previous time step past_val_errors = self.get_past_errors(index, "test") # get the output of the model (Argmax weights * expert), # each expert output and weights assigned to each expert output, expert_output, weights, true = self._process_one_batch( batch_x, batch_y, past_errors=past_val_errors, validation=True) # the choosen is the expert with higest weight by gate network _, choosen = torch.max(weights, 1) # get the error made by each expert error = self.model_assignment_error(expert_output, true) # update the error matrix with new errors note that we update with # only first time step error to prevent information leakage error = torch.Tensor( self.error_scaler.transform( error.detach().cpu().numpy().flatten().reshape(-1, 1))).reshape( -1, self.num_experts).to(self.device) self.set_past_errors(error.detach(), index, "test") if i == 0: preds = output.detach().cpu().numpy() trues = true.detach().cpu().numpy() assignment = choosen.detach().cpu().numpy() else: preds = np.concatenate((preds, output.detach().cpu().numpy()), axis=0) trues = np.concatenate((trues, true.detach().cpu().numpy()), axis=0) assignment = np.concatenate( (assignment, choosen.detach().cpu().numpy()), axis=0) if self.args.plot: assignment = np.array(assignment).squeeze() plt.figure() plt.plot(assignment) plt.title("assignment over time") plt.legend() plt.savefig("./Graphs/" + setting + "_Final_assignment.png") plt.show() plt.close() if self.args.scale: # Transform dataset back to orignal form preds = self.label_scaler.inverse_transform(preds.reshape(-1, 1)) trues = self.label_scaler.inverse_transform(trues.reshape(-1, 1)) # save predictions made by model folder_path = "./results/" + setting + "/" check_folder = os.path.isdir(folder_path) if not check_folder: os.makedirs(folder_path) np.save(folder_path + "real_prediction.npy", preds) # Evaluate the model performance mae, mse, rmse, mape, mspe = metric(preds, trues) print("mse:{}, mae:{}, rmse:{}".format(mse, mae, rmse)) self.past_test_error = torch.zeros( (len(self.data.test_x), self.num_experts), requires_grad=False).to(self.device) _, upper_bound, oracle_acc = self.get_upper_bound_accuracy( pred_loader, flag="test", plot=self.args.plot, setting=setting) return mae, mse, rmse, mape, mspe, upper_bound, oracle_acc
def get_upper_bound_accuracy(self, test_loader, hide_print=False, flag="train", plot=False, setting=None): """Function that calculates upper bound, oracle accuracy and mse. Args: test_loader: data loader hide_print: a boolean to indicate whether to print each expert accuracy flag: a string indicating dataset type plot: a boolean to indicate whether plot oracle assignment over time and expert error setting: a string containing model name Returns: mse: mean squared error for entire IME upper_bound: best possible mse had the gate network choosen the best expert oracle_acc: Oracle accuracy showing the percentage of times the gate network chooses the best expert """ self.model.eval() upper_bound = 0 expert = [0 for i in range(self.num_experts)] expert_error = [[] for i in range(self.num_experts)] for i, (batch_x, index, batch_y) in enumerate(test_loader): # get the error made by experts in each previous time step past_val_errors = self.get_past_errors(index, flag) # get the output of the model (Argmax weights * expert), # each expert output and weights assigned to each expert output, expert_output, weights, true = self._process_one_batch( batch_x, batch_y, past_errors=past_val_errors, validation=True) # get the error made by each expert error = self.model_assignment_error(expert_output, true) # update the error matrix with new errors note that we update with # only first time step error to prevent information leakage self.set_past_errors(error.detach(), index, flag) # the best accuracy would be the one that gives minimum error. _, actual_best = torch.min(error, 1) # the choosen is the expert with higest weight by gate network _, choosen = torch.max(weights, 1) upper_bound += torch.mean(torch.min(error, 1)[0]) for y in range(self.num_experts): expert[y] += torch.mean(error[:, y]).item() expert_error[y].append(torch.mean(error[:, y]).item()) # or the entire dataset get predictions and labels if i == 0: preds = output.detach().cpu().numpy() trues = true.detach().cpu().numpy() else: preds = np.concatenate((preds, output.detach().cpu().numpy()), axis=0) trues = np.concatenate((trues, true.detach().cpu().numpy()), axis=0) # for the entire dataset get best expert (with minimum error) # and choosen expert by gate network if i == 0: actual_best_expert = actual_best.detach().cpu().numpy() choosen_expert = choosen.squeeze(-1).detach().cpu().numpy() else: actual_best_expert = np.concatenate( (actual_best_expert, actual_best.detach().cpu().numpy()), axis=0) choosen_expert = np.concatenate( (choosen_expert, choosen.squeeze(-1).detach().cpu().numpy()), axis=0) # Oracle accuracy calcuates the percentage of times the gate network # chooses the best expert oracle_acc = accuracy_score(actual_best_expert, choosen_expert) upper_bound = upper_bound.item() / len(test_loader) # update past error matrix if flag == "test": self.past_test_error = torch.Tensor( self.error_scaler.transform( self.past_test_error.detach().cpu().numpy().flatten().reshape( -1, 1))).reshape(-1, self.num_experts).to(self.device) elif flag == "train": self.error_scaler.fit( self.past_train_error.detach().cpu().numpy().flatten().reshape(-1, 1)) self.past_train_error = torch.Tensor( self.error_scaler.transform( self.past_train_error.detach().cpu().numpy().flatten().reshape( -1, 1))).reshape(-1, self.num_experts).to(self.device) else: self.past_val_error = torch.Tensor( self.error_scaler.transform( self.past_val_error.detach().cpu().numpy().flatten().reshape( -1, 1))).reshape(-1, self.num_experts).to(self.device) # get accuracy metrics _, mse, _, _, _ = metric(preds, trues) if not hide_print: print(flag) print("oracle accuracy: {0:.7f}".format(oracle_acc)) print("Upper bound: {0:.7f}".format(upper_bound)) for y in range(self.num_experts): print("Expert {0}: {1:.7f}".format(y, expert[y] / len(test_loader))) print("MSE: {0:.7f}".format(mse)) print() if plot: plt.figure() plt.plot(actual_best_expert) plt.title("oracle assignment") plt.legend() plt.savefig("./Graphs/" + setting + "_oracle_assignment_" + flag + ".png") plt.show() plt.close() plt.figure() _, axs = plt.subplots(self.num_experts, 1) for y in range(self.num_experts): axs[y].plot(expert_error[y]) axs[y].set_xlabel("expert" + str(y)) plt.savefig("./Graphs/" + setting + "_expert_error_" + flag + ".png") plt.show() plt.close() return mse, upper_bound, oracle_acc