def step(self, adapted_params_list, val_tasks, is_training, additional_loss_term=None): self._optimizer.zero_grad() post_update_losses = [] for adapted_params, task in zip(adapted_params_list, val_tasks): preds = self._model(task.x, params=adapted_params) if ~is_training: preds = torch.clamp(preds, 0, 1) loss = mae(preds, task.y) else: loss = self._loss_func(preds, task.y) post_update_losses.append(loss) if additional_loss_term is None: mean_loss = torch.mean(torch.stack(post_update_losses)) else: mean_loss = torch.mean( torch.stack(post_update_losses)) + additional_loss_term if is_training: mean_loss.backward() self._optimizer.step() return mean_loss
def test(sess, model, generator, result_file): pointwise_results = [] pairwise_results = [] mae_results = [] for city in CITIES: # Validate the model on the entire validation set generator.load_test_set(sess, city) pds = [] gts = [] factors = [] for _ in trange(generator.test_batches_per_epoch[city], desc=city): batch_img, batch_label, batch_factor = generator.get_next(sess) pd = sess.run(model.prob, feed_dict={model.x: batch_img}) pds.extend(pd.tolist()) gts.extend(batch_label.tolist()) factors.extend(batch_factor.tolist()) pds = np.asarray(pds) gts = np.asarray(gts) pointwise_results.append(metrics.pointwise(pds, gts)) pairwise_results.append(metrics.pairwise(pds, gts, factors)) mae_results.append(metrics.mae(pds, gts)) layout = '{:15} {:>10} {:>10} {:>10} {:>10}' print(layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE')) result_file.write( layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE\n')) print('-' * 59) result_file.write('-' * 59 + '\n') test_sizes = [] for city, pointwise, pairwise, mae in zip(CITIES, pointwise_results, pairwise_results, mae_results): print( layout.format(city, generator.test_sizes[city], '{:.3f}'.format(pointwise), '{:.3f}'.format(pairwise), '{:.3f}'.format(mae))) result_file.write( layout.format(city, generator.test_sizes[city], '{:.3f}'.format(pointwise), '{:.3f}'.format(pairwise), '{:.3f}\n'.format(mae))) test_sizes.append(generator.test_sizes[city]) test_sizes = np.asarray(test_sizes, dtype=np.int) total = np.sum(test_sizes) avg_pointwise = np.sum(np.asarray(pointwise_results) * test_sizes) / total avg_pairwise = np.sum(np.asarray(pairwise_results) * test_sizes) / total avg_mae = np.sum(np.asarray(mae_results) * test_sizes) / total print('-' * 59) result_file.write('-' * 59 + '\n') print( layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise), '{:.3f}'.format(avg_pairwise), '{:.3f}'.format(avg_mae))) result_file.write( layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise), '{:.3f}'.format(avg_pairwise), '{:.3f}\n'.format(avg_mae))) result_file.flush()
def update(self, test_predictions, val_predictions=None, year=None): self.test_predictions = self.test_predictions.append(test_predictions) try: self.test_metrics[str(year + 2014) + '/' + str(year + 15)] = [ metrics.crps(test_predictions), metrics.nll(test_predictions), metrics.mae(test_predictions), metrics.rmse(test_predictions), metrics.smape(test_predictions), metrics.corr(test_predictions), np.ma.masked_invalid(metrics.mb_log(test_predictions)).mean(), metrics.sdp(test_predictions) ] except: pass if year == 3: self.test_metrics['Average'] = self.test_metrics.mean(1) self.test_metrics['Average'].loc['SDP'] = np.abs( self.test_metrics.loc['SDP'].values[-1]).mean() try: self.val_predictions = self.val_predictions.append(val_predictions) self.val_metrics[str(year + 2013) + '/' + str(year + 14)] = [ metrics.crps(val_predictions), metrics.nll(val_predictions), metrics.mae(val_predictions), metrics.rmse(val_predictions), metrics.smape(val_predictions), metrics.corr(val_predictions), metrics.mb_log(val_predictions).mean(), metrics.sdp(val_predictions) ] except: pass self.val_metrics['Average'] = self.val_metrics.mean(1) self.val_metrics['Average'].loc['SDP'] = np.abs( self.val_metrics.loc['SDP'].values[-1]).mean() self.test_metrics['Average'] = self.test_metrics.mean(1) self.test_metrics['Average'].loc['SDP'] = np.abs( self.test_metrics.loc['SDP'].values[-1]).mean()
def on_epoch_end(self, epoch, logs={}): train_predict = np.asarray(self.model.predict(self.train_x)) train_predictions_indices = [ example_pred_probs.tolist().index(max(example_pred_probs)) for example_pred_probs in train_predict ] train_predictions = [ self.labels[prediction] for prediction in train_predictions_indices ] train_targets = [self.labels[target] for target in self.train_y] logs['macro_f1'] = macro_f1(train_targets, train_predictions) logs['macro_recall'] = macro_recall(train_targets, train_predictions) logs['mae'] = mae(train_targets, train_predictions) logs['macro_averaged_mae'] = macro_averaged_mae( train_targets, train_predictions) val_data = self.validation_data[:self.num_inputs] val_predict = np.asarray(self.model.predict(val_data)) predictions_indices = [ example_pred_probs.tolist().index(max(example_pred_probs)) for example_pred_probs in val_predict ] predictions = [ self.labels[prediction] for prediction in predictions_indices ] val_targ = flatten(self.validation_data[self.num_inputs]) targets = [self.labels[target] for target in val_targ] logs['val_macro_f1'] = macro_f1(targets, predictions) logs['val_macro_recall'] = macro_recall(targets, predictions) logs['val_mae'] = mae(targets, predictions) logs['val_macro_averaged_mae'] = macro_averaged_mae( targets, predictions)
def main(): # dataset has format like [user_id, song_id, play_count] file = 'train_triplets.txt' print("Loading data...") load_data(file) print("Starting evaluation...") calc_neighbours() print("Finished evaluations.") print_top_songs_for_user(1) print("Starting cross validation...") print("RMSE result: ", str(rmse(train_set, test_set))) print("MAE result: ", str(mae(train_set, test_set))) print("NDCG result: ", str(ndcg(train_set, test_set)))
def evaluate(self, observed, estimated, zone, res = 0.5): ''' Returns evaluation between observed and estimated rainfall maps by computing following metrics: ['BIAS', 'CORREALTION', 'Nash-Sutcliffe', 'RMSE', 'MAE', 'MEAN_OBS','MEAN_EST'] Inputs: observed - 2D array. Observed rainfall map. estimated - 2D array. Estimated rainfall map. zone - (2,2) tuple. Evaluation study zone [km x km] Optional: res - scalar. Resolution for comparison, [km]. Outputs: metrics - dictionary. Statistical metrics: ['bias', 'corr', 'nash', 'rmse', 'mae', 'mean_obs','mean_est'] ''' # We neglect area that is not estimated for comparison purpose. estimated[estimated <= -999] = -999 observed[estimated <= -999] = -999 ((x0, x1), (y0, y1)) = zone # Cut the zone for evaluation t1, t2, t3, t4 = int(y0/res), int(y1/res), int(x0/res), int(x1/res) observed = observed[t1:t2, t3:t4] estimated = estimated[t1:t2, t3:t4] est = estimated[estimated<>-999].flatten() obs = observed[observed<>-999].flatten() stats = dict() stats['bias'] = metrics.bias(obs, est) stats['corr'] = metrics.corr(obs, est) stats['nash'] = metrics.nash(obs, est) stats['rmse'] = metrics.rmse(obs, est) stats['mae'] = metrics.mae(obs, est) stats['mean_obs'] = metrics.average(obs) stats['mean_est'] = metrics.average(est) # additional metrics can be added ##stats['likelihood'] = metrics.likelihood(obs, est) ##stats['mape'] = metrics.mape(obs, est) ##stats['mse'] = metrics.mse(obs, est) ##stats['mspe'] = metrics.mspe(obs, est) ##stats['rmspe'] = metrics.rmspe(obs, est) return stats
def plot(listbox, event): selected_files = [listbox.get(i) for i in listbox.curselection()] max_loc = None for name in selected_files: data = np.genfromtxt(name, delimiter=',', skip_header=1) y_pred = data[:, 1] y_true = data[:, 0] label = '{0} MAE:{1:.2f}'.format( os.path.splitext(os.path.basename(name))[0], metrics.mae(y_true, y_pred)) if max_loc is None: max_loc = np.argmax(y_true) plt.plot(y_true, label='True', color='black') plt.plot(y_pred, label=label) else: offset = np.argmax(y_true) - max_loc plt.plot(np.arange(len(y_pred)) - offset, y_pred, label=label) plt.legend() plt.grid() plt.show()
def main(args): meta_info = { "POLLUTION": [5, 50, 14], "HR": [32, 50, 13], "BATTERY": [20, 50, 3] } output_directory = "output/" verbose = True batch_size = 64 freeze_model_flag = True params = {'batch_size': batch_size, 'shuffle': True, 'num_workers': 0} dataset_name = args.dataset model_name = args.model learning_rate = args.learning_rate save_model_file = args.save_model_file load_model_file = args.load_model_file lower_trial = args.lower_trial upper_trial = args.upper_trial is_test = args.is_test epochs = args.epochs experiment_id = args.experiment_id adaptation_steps = args.adaptation_steps assert model_name in ("FCN", "LSTM"), "Model was not correctly specified" assert dataset_name in ("POLLUTION", "HR", "BATTERY") window_size, task_size, input_dim = meta_info[dataset_name] batch_size = 64 train_data = pickle.load( open( "../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) train_data_ML = pickle.load( open( "../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) validation_data = pickle.load( open( "../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) validation_data_ML = pickle.load( open( "../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) test_data = pickle.load( open( "../../Data/TEST-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) test_data_ML = pickle.load( open( "../../Data/TEST-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) if is_test == 0: test_data = validation_data train_idx, val_idx, test_idx = split_idx_50_50( test_data.file_idx) if is_test else split_idx_50_50( validation_data.file_idx) n_domains_in_test = np.max(test_data.file_idx) + 1 test_loss_list = [] initial_test_loss_list = [] trials_loss_list = [] #trial = 0 for trial in range(lower_trial, upper_trial): output_directory = "../../Models/" + dataset_name + "_" + model_name + "_MAML/" + str( trial) + "/" #save_model_file_ = output_directory + "encoder_"+save_model_file #save_model_file_2 = output_directory + save_model_file save_model_file_ = output_directory + experiment_id + "_encoder_model.pt" save_model_file_2 = output_directory + experiment_id + "_model.pt" load_model_file_ = output_directory + load_model_file model = LSTMModel(batch_size=batch_size, seq_len=window_size, input_dim=input_dim, n_layers=2, hidden_dim=120, output_dim=1) model2 = nn.Linear(120, 1) model.cuda() model2.cuda() maml = l2l.algorithms.MAML(model2, lr=learning_rate, first_order=False) model.load_state_dict(torch.load(save_model_file_)) maml.load_state_dict(torch.load(save_model_file_2)) n_domains_in_test = np.max(test_data.file_idx) + 1 error_list = [] y_list = [] for domain in range(n_domains_in_test): x_test = test_data.x y_test = test_data.y temp_train_data = SimpleDataset( x=np.concatenate([ x_test[np.concatenate([train_idx[domain], val_idx[domain]])][np.newaxis, :], x_test[test_idx[domain]][np.newaxis, :] ]), y=np.concatenate([ y_test[np.concatenate([train_idx[domain], val_idx[domain]])][np.newaxis, :], y_test[test_idx[domain]][np.newaxis, :] ])) total_tasks_test = len(test_data_ML) learner = maml.clone() # Creates a clone of model learner.cuda() accum_error = 0.0 accum_std = 0.0 count = 0.0 input_dim = test_data_ML.x.shape[-1] window_size = test_data_ML.x.shape[-2] output_dim = test_data_ML.y.shape[-1] task = 0 model2 = nn.Linear(120, 1) model2.load_state_dict(copy.deepcopy(maml.module.state_dict())) model.cuda() model2.cuda() x_spt, y_spt = temp_train_data[task] x_qry = temp_train_data.x[(task + 1)] y_qry = temp_train_data.y[(task + 1)] if model_name == "FCN": x_qry = np.transpose(x_qry, [0, 2, 1]) x_spt = np.transpose(x_spt, [0, 2, 1]) x_spt, y_spt = to_torch(x_spt), to_torch(y_spt) x_qry = to_torch(x_qry) y_qry = to_torch(y_qry) opt2 = optim.SGD(list(model2.parameters()), lr=learning_rate) #learner.module.train() size_back = 300 step_size = task_size * size_back #model2.eval() for step in range(adaptation_steps): print(step) #model2.train() for idx in range(x_spt.shape[0] - task_size * size_back, x_spt.shape[0], step_size): pred = model2(model.encoder(x_spt[idx:idx + step_size])) print(pred.shape) print(step_size) error = mae(pred, y_spt[idx:idx + step_size]) print(error) opt2.zero_grad() error.backward() #learner.adapt(error) opt2.step() #model2.eval() #learner.module.eval() step = x_qry.shape[0] // 255 for idx in range(0, x_qry.shape[0], step): pred = model2(model.encoder(x_qry[idx:idx + step])) error = mae(pred, y_qry[idx:idx + step]) accum_error += error.data accum_std += error.data**2 count += 1 error = accum_error / count y_list.append(y_qry.cpu().numpy()) error_list.append(float(error.cpu().numpy())) print(np.mean(error_list)) print(error_list) trials_loss_list.append(np.mean(error_list)) print("mean:", np.mean(trials_loss_list)) print("std:", np.std(trials_loss_list))
import datetime from data import load_data, get_data, get_song_sets from model import learning from metrics import rmse, mae, ndcg max_users = 1000 iterations = 5 print("Data loading...") load_data(max_users) print("Finished.") # Cross validation for i in range(0, iterations): print("Iteration", i + 1, "/", iterations) print("Learning is in process...") learning_set, testing_set = get_song_sets() data = get_data() start_time = time.time() learning(data, learning_set, 100) finish_time = time.time() print("Learning finished. Time:", datetime.timedelta(seconds=finish_time - start_time)) print("RMSE:", rmse(data, testing_set)) print("MAE: ", mae(data, testing_set)) print("NDCG:", ndcg(data, testing_set)) print("=====")
def main(args): meta_info = { "POLLUTION": [5, 50, 14], "HR": [32, 50, 13], "BATTERY": [20, 50, 3] } output_directory = "output/" verbose = True batch_size = 64 freeze_model_flag = True params = {'batch_size': batch_size, 'shuffle': True, 'num_workers': 0} dataset_name = args.dataset model_name = args.model learning_rate = args.learning_rate save_model_file = args.save_model_file load_model_file = args.load_model_file lower_trial = args.lower_trial upper_trial = args.upper_trial is_test = args.is_test epochs = args.epochs experiment_id = args.experiment_id adaptation_steps = args.adaptation_steps assert model_name in ("FCN", "LSTM"), "Model was not correctly specified" assert dataset_name in ("POLLUTION", "HR", "BATTERY") window_size, task_size, input_dim = meta_info[dataset_name] batch_size = 64 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") loss_fn = mae train_data = pickle.load( open( "../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) train_data_ML = pickle.load( open( "../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) validation_data = pickle.load( open( "../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) validation_data_ML = pickle.load( open( "../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) test_data = pickle.load( open( "../../Data/TEST-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) test_data_ML = pickle.load( open( "../../Data/TEST-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) # paramters wto increase capactiy of the model n_layers_task_net = 2 n_layers_task_encoder = 1 n_layers_task_decoder = 1 hidden_dim_task_net = 120 hidden_dim_encoder = 120 hidden_dim_decoder = 120 input_dim_task_net = input_dim input_dim_task_encoder = input_dim + 1 output_dim_task_net = 1 output_dim_task_decoder = input_dim + 1 output_dim = 1 if is_test == 0: test_data = validation_data train_idx, val_idx, test_idx = split_idx_50_50( test_data.file_idx) if is_test else split_idx_50_50( validation_data.file_idx) n_domains_in_test = np.max(test_data.file_idx) + 1 test_loss_list = [] initial_test_loss_list = [] trials_loss_list = [] modulate_task_net = True #trial = 0 for trial in range(lower_trial, upper_trial): output_directory = "../../Models/" + dataset_name + "_" + model_name + "_MMAML/" + str( trial) + "/" #save_model_file_ = output_directory + "encoder_"+save_model_file #save_model_file_2 = output_directory + save_model_file save_model_file_encoder = output_directory + experiment_id + "_encoder_model.pt" save_model_file_ = output_directory + experiment_id + "_model.pt" load_model_file_ = output_directory + load_model_file ##creating the network task_net = LSTMModel(batch_size=batch_size, seq_len=window_size, input_dim=input_dim_task_net, n_layers=n_layers_task_net, hidden_dim=hidden_dim_task_net, output_dim=output_dim_task_net) task_encoder = LSTMModel(batch_size=batch_size, seq_len=task_size, input_dim=input_dim_task_encoder, n_layers=n_layers_task_encoder, hidden_dim=hidden_dim_encoder, output_dim=1) task_decoder = LSTMDecoder(batch_size=1, n_layers=n_layers_task_decoder, seq_len=task_size, output_dim=output_dim_task_decoder, hidden_dim=hidden_dim_encoder, latent_dim=hidden_dim_decoder, device=device) lmbd = Lambda(hidden_dim_encoder, hidden_dim_task_net) multimodal_learner = MultimodalLearner(task_net, task_encoder, task_decoder, lmbd, modulate_task_net) multimodal_learner.to(device) output_layer = nn.Linear(120, 1) output_layer.to(device) maml = l2l.algorithms.MAML(output_layer, lr=learning_rate, first_order=False) multimodal_learner.load_state_dict(torch.load(save_model_file_encoder)) maml.load_state_dict(torch.load(save_model_file_)) n_domains_in_test = np.max(test_data.file_idx) + 1 error_list = [] y_list = [] for domain in range(n_domains_in_test): print("Domain:", domain) x_test = test_data.x y_test = test_data.y temp_train_data = SimpleDataset( x=np.concatenate([ x_test[np.concatenate([train_idx[domain], val_idx[domain]])][np.newaxis, :], x_test[test_idx[domain]][np.newaxis, :] ]), y=np.concatenate([ y_test[np.concatenate([train_idx[domain], val_idx[domain]])][np.newaxis, :], y_test[test_idx[domain]][np.newaxis, :] ])) total_tasks_test = len(test_data_ML) learner = maml.clone() # Creates a clone of model learner.cuda() accum_error = 0.0 accum_std = 0.0 count = 0.0 input_dim = test_data_ML.x.shape[-1] window_size = test_data_ML.x.shape[-2] output_dim = test_data_ML.y.shape[-1] task_id = 0 #model2 = nn.Linear(120, 1) #model2.load_state_dict(copy.deepcopy(maml.module.state_dict())) #model.cuda() #model2.cuda() output_layer = nn.Linear(120, 1) output_layer.load_state_dict( copy.deepcopy(maml.module.state_dict())) output_layer.to(device) x_spt, y_spt = temp_train_data[task_id] x_qry = temp_train_data.x[(task_id + 1)] y_qry = temp_train_data.y[(task_id + 1)] task = get_task_encoder_input( SimpleDataset(x=x_spt[-50:][np.newaxis, :], y=y_spt[-50:][np.newaxis, :])) task = to_torch(task) if model_name == "FCN": x_qry = np.transpose(x_qry, [0, 2, 1]) x_spt = np.transpose(x_spt, [0, 2, 1]) x_spt, y_spt = to_torch(x_spt), to_torch(y_spt) x_qry = to_torch(x_qry) y_qry = to_torch(y_qry) opt2 = optim.SGD(list(output_layer.parameters()), lr=learning_rate) #learner.module.train() size_back = 200 step_size = task_size * size_back multimodal_learner.train() #model2.eval() for step in range(adaptation_steps): step_size = 1 error_accum = 0 count = 0 #model2.train() for idx in range(0, x_spt.shape[0], step_size): x_spt_encoding, (vrae_loss, _, _) = multimodal_learner( x_spt[idx:idx + step_size], task, output_encoding=True) pred = output_layer(x_spt_encoding) error_accum += mae(pred, y_spt[idx:idx + step_size]) count += 1 opt2.zero_grad() error = error_accum / count error.backward() #learner.adapt(error) opt2.step() #model2.eval() #learner.module.eval() multimodal_learner.eval() step = x_qry.shape[0] // 255 error_accum = 0 count = 0 for idx in range(0, x_qry.shape[0], step): x_qry_encoding, (vrae_loss, _, _) = multimodal_learner(x_qry[idx:idx + step], task, output_encoding=True) pred = output_layer(x_qry_encoding) error = mae(pred, y_qry[idx:idx + step]) accum_error += error.data accum_std += error.data**2 count += 1 error = accum_error / count y_list.append(y_qry.cpu().numpy()) error_list.append(float(error.cpu().numpy())) print(np.mean(error_list)) print(error_list) trials_loss_list.append(np.mean(error_list)) print("mean:", np.mean(trials_loss_list)) print("std:", np.std(trials_loss_list))
def main(_): _, features, labels = load_data( os.path.join(FLAGS.data_dir, FLAGS.dataset, 'train.h5')) # Training model = GaussianNB() model.fit(features, labels) # Evaluation pointwise_results = [] pairwise_results = [] mae_results = [] test_sizes = [] for city in CITIES: factors, features, labels = load_data( os.path.join(FLAGS.data_dir, FLAGS.dataset, 'val_{}.h5'.format(city))) pd_probs = model.predict_proba(features) onehot_labels = to_onehot(labels) pointwise_results.append( metrics.pointwise(pds=pd_probs, gts=onehot_labels)) pairwise_results.append( metrics.pairwise(pds=pd_probs, gts=onehot_labels, factors=factors)) mae_results.append(metrics.mae(pds=pd_probs, gts=onehot_labels)) test_sizes.append(len(labels)) result_file = open('result_{}_nb.txt'.format(FLAGS.dataset), 'w') layout = '{:15} {:>10} {:>10} {:>10} {:>10}' print(layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE')) result_file.write( layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE\n')) print('-' * 59) result_file.write('-' * 59 + '\n') for city, pointwise, pairwise, mae, tsize in zip(CITIES, pointwise_results, pairwise_results, mae_results, test_sizes): print( layout.format(city, tsize, '{:.3f}'.format(pointwise), '{:.3f}'.format(pairwise), '{:.3f}'.format(mae))) result_file.write( layout.format(city, tsize, '{:.3f}'.format(pointwise), '{:.3f}'.format(pairwise), '{:.3f}\n'.format(mae))) test_sizes = np.asarray(test_sizes, dtype=np.int) total = np.sum(test_sizes) avg_pointwise = np.sum(np.asarray(pointwise_results) * test_sizes) / total avg_pairwise = np.sum(np.asarray(pairwise_results) * test_sizes) / total avg_mae = np.sum(np.asarray(mae_results) * test_sizes) / total print('-' * 59) result_file.write('-' * 59 + '\n') print( layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise), '{:.3f}'.format(avg_pairwise), '{:.3f}'.format(avg_mae))) result_file.write( layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise), '{:.3f}'.format(avg_pairwise), '{:.3f}\n'.format(avg_mae))) result_file.close()
def test(loss_fn, maml, multimodal_model, task_data, dataset_name, data_ML, adaptation_steps, learning_rate, noise_level, noise_type, is_test = True, horizon = 10): total_tasks = len(data_ML) task_size = data_ML.x.shape[-3] input_dim = data_ML.x.shape[-1] window_size = data_ML.x.shape[-2] output_dim = data_ML.y.shape[-1] if is_test: step = total_tasks//100 else: step = 1 step = 1 if step == 0 else step grid = [0., noise_level] accum_error = 0.0 count = 1.0 for task_idx in range(0, (total_tasks-horizon-1), step): temp_file_idx = data_ML.file_idx[task_idx:task_idx+horizon+1] if(len(np.unique(temp_file_idx))>1): continue learner = maml.clone() x_spt, y_spt = data_ML[task_idx] x_qry = data_ML.x[(task_idx+1):(task_idx+1+horizon)].reshape(-1, window_size, input_dim) y_qry = data_ML.y[(task_idx+1):(task_idx+1+horizon)].reshape(-1, output_dim) task = task_data[task_idx:task_idx+1].cuda() x_spt, y_spt = to_torch(x_spt), to_torch(y_spt) x_qry = to_torch(x_qry) y_qry = to_torch(y_qry) epsilon = grid[np.random.randint(0,len(grid))] if noise_type == "additive": y_spt = y_spt+epsilon y_qry = y_qry+epsilon else: y_spt = y_spt*(1+epsilon) y_qry = y_qry*(1+epsilon) for step in range(adaptation_steps): x_encoding, _ = multimodal_model(x_spt, task, output_encoding=True) pred = learner(x_encoding) error = loss_fn(pred, y_spt) learner.adapt(error) x_encoding, _ = multimodal_model(x_qry, task, output_encoding=True) y_pred = learner(x_encoding) y_pred = torch.clamp(y_pred, 0, 1) error = mae(y_pred, y_qry) accum_error += error.data count += 1 error = accum_error/count return error.cpu().numpy()
if __name__ == "__main__": # Matrix of movie ratings train_data, test_data = get_movie_matrix() # Get all user mean values user_mean = get_user_mean(train_data) start = time.time() prediction_matrix = pd.DataFrame(index=train_data.index) for name, data in train_data.iteritems(): prediction_matrix[name] = main(train_data, name, k) # break logging.info("Process done in: {0:.2f} seconds".format(time.time() - start)) inter_columns = np.intersect1d(prediction_matrix.columns.values, test_data.columns.values) small_pred = prediction_matrix[inter_columns].dropna(how='all') small_test = test_data.loc[:, inter_columns].dropna(how='all') print("Test Matrix\n", small_test) print("Predicted Matrix\n", small_pred.loc[small_test.index, :]) logging.info('\nMetric Calculations RMSE and MAE') rmse_value = metrics.rmse(test_data, prediction_matrix) print(f'RMSE:\t{rmse_value}') mae_value = metrics.mae(test_data, prediction_matrix) print(f'MAE:\t{mae_value}')
# Preprocessing X = shuffled.iloc[:, :-1].squeeze() y = (shuffled.iloc[:, -1:]).T.squeeze() len_estate = len(y) # Splitting data X_train, y_train = X.loc[:split*len_estate], y.loc[:split*len_estate] X_test, y_test = X.loc[split*len_estate+1:].reset_index( drop=True), y.loc[split*len_estate+1:].reset_index(drop=True) # Learning tree print("Please wait for some time, it takes time, you can change max depth if it takes too long time.") tree = DecisionTree(criterion="information_gain", max_depth=max_depth) tree.fit(X_train, y_train) tree.plot() # Printing accuracies for different depths for depth in range(2, max_depth+1): y_hat = tree.predict(X_test, max_depth=depth) print("Depth: ", depth) print('\tRMSE: ', rmse(y_hat, y_test)) print('\tMAE: ', mae(y_hat, y_test)) # Decision Tree Regressor from Sci-kit learn dt = DecisionTreeRegressor(random_state=0) dt.fit(X_train, y_train) y_hat = pd.Series(dt.predict(X_test)) print('Sklearn RMSE: ', rmse(y_hat, y_test)) print('Sklearn MAE: ', mae(y_hat, y_test))
def test2(maml, model, model_name, dataset_name, test_data_ML, adaptation_steps, learning_rate, noise_level, noise_type, is_test = True, horizon = 10): total_tasks_test = len(test_data_ML) error_list = [] learner = maml.clone() # Creates a clone of model learner.cuda() accum_error = 0.0 accum_std = 0.0 count = 0.0 grid = [0., noise_level] input_dim = test_data_ML.x.shape[-1] window_size = test_data_ML.x.shape[-2] output_dim = test_data_ML.y.shape[-1] if is_test: step = total_tasks_test//100 else: step = 1 step = 1 if step == 0 else step for task in range(0, (total_tasks_test-horizon-1), step): temp_file_idx = test_data_ML.file_idx[task:task+horizon+1] if(len(np.unique(temp_file_idx))>1): continue if model_name == "LSTM": model2 = LSTMModel( batch_size=None, seq_len = None, input_dim = input_dim, n_layers = 2, hidden_dim = 120, output_dim =1) elif model_name == "LSTM_MRA": model2 = LSTMModel_MRA( batch_size=None, seq_len = window_size, input_dim = input_dim, n_layers = 2, hidden_dim = 120, output_dim =1) elif model_name == "FCN": kernels = [8,5,3] if window_size != 5 else [4,2,1] model2 = FCN(time_steps = window_size, channels=[input_dim, 128, 128, 128] , kernels=kernels) #model2.cuda() #model2.load_state_dict(copy.deepcopy(maml.module.state_dict())) #opt2 = optim.Adam(model2.parameters(), lr=learning_rate) learner = maml.clone() x_spt, y_spt = test_data_ML[task] x_qry = test_data_ML.x[(task+1):(task+1+horizon)].reshape(-1, window_size, input_dim) y_qry = test_data_ML.y[(task+1):(task+1+horizon)].reshape(-1, output_dim) #x_qry = test_data_ML.x[(task+1)].reshape(-1, window_size, input_dim) #y_qry = test_data_ML.y[(task+1)].reshape(-1, output_dim) if model_name == "FCN": x_qry = np.transpose(x_qry, [0,2,1]) x_spt = np.transpose(x_spt, [0,2,1]) x_spt, y_spt = to_torch(x_spt), to_torch(y_spt) x_qry = to_torch(x_qry) y_qry = to_torch(y_qry) epsilon = grid[np.random.randint(0,len(grid))] if noise_type == "additive": y_spt = y_spt+epsilon y_qry = y_qry+epsilon else: y_spt = y_spt*(1+epsilon) y_qry = y_qry*(1+epsilon) #learner.module.train() #model2.eval() for step in range(adaptation_steps): #model2.train() pred = learner(model.encoder(x_spt)) error = mae(pred, y_spt) #opt2.zero_grad() #error.backward() learner.adapt(error) #opt2.step() #model2.eval() #learner.module.eval() y_pred = learner(model.encoder(x_qry)) y_pred = torch.clamp(y_pred, 0, 1) error = mae(y_pred, y_qry) accum_error += error.data accum_std += error.data**2 count += 1 error = accum_error/count print("std:", accum_std/count) return error
def benchmark_test(n_train, n_test, n_val, bm): (mae_is0_is0, inversion_is0_is0, plateau_is0_is0, dist_const_is0_is0) = [], [], [], [] (mae_is0_avg, inversion_is0_avg, plateau_is0_avg, dist_const_is0_avg) = [], [], [], [] (mae_avg_is0, inversion_avg_is0, plateau_avg_is0, dist_const_avg_is0) = [], [], [], [] (mae_avg_avg, inversion_avg_avg, plateau_avg_avg, dist_const_avg_avg) = [], [], [], [] tot_couples_train, tot_couples_test = [], [] for i in range(niter): print("Experiment n. {}".format(i)) # reading i-th test and trainig set df_train, df_test, df_val = reader.read_replicable_dataset( n_train, n_test, bm, i) ninput = len(list(df_train.filter(regex='var_*'))) width = ninput * 10 # normalizing test and training set scaler = preprocessing.MinMaxScaler() label_target = suff_label_target + str(0) if improved_dataset == 1: add_features = ed.getAdditionalFeatures(bm) for add_feat in add_features: df_train['{}-{}'.format( add_feat[0], add_feat[1]) ] = df_train[add_feat[0]] - df_train[add_feat[1]] df_test['{}-{}'.format( add_feat[0], add_feat[1]) ] = df_test[add_feat[0]] - df_test[add_feat[1]] df_val['{}-{}'.format( add_feat[0], add_feat[1]) ] = df_val[add_feat[0]] - df_val[add_feat[1]] ninput += len(add_features) #Arrange the df with err column at the end for _label_target in label_targets: df_temp = df_train.pop(_label_target) df_train[_label_target] = df_temp df_temp = df_test.pop(_label_target) df_test[_label_target] = df_temp df_temp = df_val.pop(_label_target) df_val[_label_target] = df_temp if cap_error: # capping error for _label_target in label_targets: df_train[_label_target] = [cap if x > cap else x for x in df_train[_label_target]] df_test[_label_target] = [cap if x > cap else x for x in df_test[_label_target]] df_val[_label_target] = [cap if x > cap else x for x in df_val[_label_target]] # -log10(err) for _label_target in label_targets: df_train[_label_target] = [sys.float_info.min if 0 == x else -np.log10(x) for x in df_train[_label_target]] df_test[_label_target] = [sys.float_info.min if 0 == x else -np.log10(x) for x in df_test[_label_target]] df_val[_label_target] = [sys.float_info.min if 0 == x else -np.log10(x) for x in df_val[_label_target]] y_avg_train = np.mean(np.array(df_train[label_targets]), axis=1, dtype='float32').reshape((-1, 1)) y_avg_test = np.mean(np.array(df_test[label_targets]), axis=1, dtype='float32').reshape((-1, 1)) y_avg_val = np.mean(np.array(df_val[label_targets]), axis=1, dtype='float32').reshape((-1, 1)) # splitting training and test set in features and targets x_train = scaler.fit_transform(df_train.iloc[:,0:ninput]) y_train = scaler.fit_transform(np.array(df_train[label_target] ).reshape((-1, 1))) x_test = scaler.fit_transform(df_test.iloc[:,0:ninput]) y_test = scaler.fit_transform(np.array(df_test[label_target] ).reshape((-1, 1))) x_val = scaler.fit_transform(df_val.iloc[:,0:ninput]) y_val = scaler.fit_transform(np.array(df_val[label_target] ).reshape((-1, 1))) (_, _, _, couples_train) = reader.remove_const_violations( x_train, y_train, y_avg_train) (_, _, _, couples_test) = reader.remove_const_violations( x_test, y_test, y_avg_test) (_, _, _, couples_val) = reader.remove_const_violations( x_val, y_val, y_avg_val) tot_couples_train.append(couples_train) tot_couples_test.append(couples_test) # Train model on specific input set automl = autosklearn.regression.AutoSklearnRegressor( time_left_for_this_task=120, per_run_time_limit=30,) automl.fit(x_train, y_train) # test it on average err among y_pred = np.array(automl.predict(x_test)) mae_is0_avg.append(metrics.mae(y_avg_test, y_pred)) tmp = metrics.const_violation(x_test, y_pred) inversion_is0_avg.append(tmp[0]) plateau_is0_avg.append(tmp[1]) dist_const_is0_avg.append(metrics.error_average_distance(y_pred)) # test it on err of same input set as training mae_is0_is0.append(metrics.mae(y_test, y_pred)) inversion_is0_is0.append(tmp[0]) plateau_is0_is0.append(tmp[1]) dist_const_is0_is0.append(metrics.error_average_distance(y_pred)) # Train model on avg error among input set automl = autosklearn.regression.AutoSklearnRegressor( time_left_for_this_task=120, per_run_time_limit=30,) automl.fit(x_train, y_avg_train) # test it on average error y_pred = np.array(automl.predict(x_test)) mae_avg_avg.append(metrics.mae(y_avg_test, y_pred)) tmp = metrics.const_violation(x_test, y_pred) inversion_avg_avg.append(tmp[0]) plateau_avg_avg.append(tmp[1]) dist_const_avg_avg.append(metrics.error_average_distance(y_pred)) # test it on error of specific input set mae_avg_is0.append(metrics.mae(y_test, y_pred)) inversion_avg_is0.append(tmp[0]) plateau_avg_is0.append(tmp[1]) dist_const_avg_is0.append(metrics.error_average_distance(y_pred)) return (mae_is0_is0, inversion_is0_is0, plateau_is0_is0, dist_const_is0_is0, mae_is0_avg, inversion_is0_avg, plateau_is0_avg, dist_const_is0_avg, mae_avg_is0, inversion_avg_is0, plateau_avg_is0, dist_const_avg_is0, mae_avg_avg, inversion_avg_avg, plateau_avg_avg, dist_const_avg_avg, tot_couples_train, tot_couples_test)
def test2(maml, model_name, test_data_ML, adaptation_steps, learning_rate, with_early_stopping=False, horizon=10): total_tasks_test = len(test_data_ML) error_list = [] learner = maml.clone() # Creates a clone of model learner.cuda() accum_error = 0.0 count = 0 input_dim = test_data_ML.x.shape[-1] window_size = test_data_ML.x.shape[-2] output_dim = test_data_ML.y.shape[-1] for task in range(0, (total_tasks_test - horizon - 1), total_tasks_test // 100): temp_file_idx = test_data_ML.file_idx[task:task + horizon + 1] if (len(np.unique(temp_file_idx)) > 1): continue if model_name == "LSTM": model2 = LSTMModel(batch_size=None, seq_len=None, input_dim=input_dim, n_layers=2, hidden_dim=120, output_dim=1) elif model_name == "FCN": kernels = [8, 5, 3] if window_size != 5 else [4, 2, 1] model2 = FCN(time_steps=window_size, channels=[input_dim, 128, 128, 128], kernels=kernels) #model2.cuda() #model2.load_state_dict(copy.deepcopy(maml.module.state_dict())) #opt2 = optim.Adam(model2.parameters(), lr=learning_rate) learner = maml.clone() x_spt, y_spt = test_data_ML[task] x_qry = test_data_ML.x[(task + 1):(task + 1 + horizon)].reshape( -1, window_size, input_dim) y_qry = test_data_ML.y[(task + 1):(task + 1 + horizon)].reshape( -1, output_dim) if model_name == "FCN": x_qry = np.transpose(x_qry, [0, 2, 1]) x_spt = np.transpose(x_spt, [0, 2, 1]) x_spt, y_spt = to_torch(x_spt), to_torch(y_spt) x_qry = to_torch(x_qry) y_qry = to_torch(y_qry) early_stopping = EarlyStopping(patience=2, model_file="temp/temp_file.pt", verbose=True) #learner.module.train() #model2.eval() for step in range(adaptation_steps): #model2.train() pred = learner(x_spt) error = mae(pred, y_spt) #opt2.zero_grad() #error.backward() learner.adapt(error) #opt2.step() if with_early_stopping: with torch.no_grad(): model2.load_state_dict( copy.deepcopy(learner.module.state_dict())) #model2.eval() pred = model2(x_qry) error = mae(pred, y_qry) early_stopping(error, model2) if early_stopping.early_stop: print("Early stopping") break if with_early_stopping: model2.load_state_dict(torch.load("temp/temp_file.pt")) #model2.eval() #learner.module.eval() pred = learner(x_qry) error = mae(pred, y_qry) accum_error += error.data count += 1 error = accum_error / count return error
def benchmark_test(n_train, n_test, n_val, bm, n_layers, neurons_per_layer): (mae_is0_is0, inversion_is0_is0, plateau_is0_is0, dist_const_is0_is0) = [], [], [], [] (mae_is0_avg, inversion_is0_avg, plateau_is0_avg, dist_const_is0_avg) = [], [], [], [] (mae_avg_is0, inversion_avg_is0, plateau_avg_is0, dist_const_avg_is0) = [], [], [], [] (mae_avg_avg, inversion_avg_avg, plateau_avg_avg, dist_const_avg_avg) = [], [], [], [] tot_couples_train, tot_couples_test = [], [] for i in range(niter): print("Experiment n. {}".format(i)) # reading i-th test and trainig set df_train, df_test, df_val = reader.read_replicable_dataset( n_train, n_test, bm, i) ninput = len(list(df_train.filter(regex='var_*'))) width = ninput * 10 # normalizing test and training set scaler = preprocessing.MinMaxScaler() label_target = suff_label_target + str(0) if improved_dataset == 1: add_features = ed.getAdditionalFeatures(bm) for add_feat in add_features: df_train['{}-{}'.format( add_feat[0], add_feat[1] )] = df_train[add_feat[0]] - df_train[add_feat[1]] df_test['{}-{}'.format( add_feat[0], add_feat[1])] = df_test[add_feat[0]] - df_test[add_feat[1]] df_val['{}-{}'.format( add_feat[0], add_feat[1])] = df_val[add_feat[0]] - df_val[add_feat[1]] ninput += len(add_features) #Arrange the df with err column at the end for _label_target in label_targets: df_temp = df_train.pop(_label_target) df_train[_label_target] = df_temp df_temp = df_test.pop(_label_target) df_test[_label_target] = df_temp df_temp = df_val.pop(_label_target) df_val[_label_target] = df_temp if cap_error: # capping error for _label_target in label_targets: df_train[_label_target] = [ cap if x > cap else x for x in df_train[_label_target] ] df_test[_label_target] = [ cap if x > cap else x for x in df_test[_label_target] ] df_val[_label_target] = [ cap if x > cap else x for x in df_val[_label_target] ] # -log10(err) for _label_target in label_targets: df_train[_label_target] = [ sys.float_info.min if 0 == x else -np.log10(x) for x in df_train[_label_target] ] df_test[_label_target] = [ sys.float_info.min if 0 == x else -np.log10(x) for x in df_test[_label_target] ] df_val[_label_target] = [ sys.float_info.min if 0 == x else -np.log10(x) for x in df_val[_label_target] ] print(df_train) sys.exit() y_avg_train = np.mean(np.array(df_train[label_targets]), axis=1, dtype='float32').reshape((-1, 1)) y_avg_test = np.mean(np.array(df_test[label_targets]), axis=1, dtype='float32').reshape((-1, 1)) y_avg_val = np.mean(np.array(df_val[label_targets]), axis=1, dtype='float32').reshape((-1, 1)) # splitting training and test set in features and targets x_train = scaler.fit_transform(df_train.iloc[:, 0:ninput]) y_train = scaler.fit_transform( np.array(df_train[label_target]).reshape((-1, 1))) x_test = scaler.fit_transform(df_test.iloc[:, 0:ninput]) y_test = scaler.fit_transform( np.array(df_test[label_target]).reshape((-1, 1))) x_val = scaler.fit_transform(df_val.iloc[:, 0:ninput]) y_val = scaler.fit_transform( np.array(df_val[label_target]).reshape((-1, 1))) (_, _, _, couples_train) = reader.remove_const_violations( x_train, y_train, y_avg_train) (_, _, _, couples_test) = reader.remove_const_violations( x_test, y_test, y_avg_test) (_, _, _, couples_val) = reader.remove_const_violations(x_val, y_val, y_avg_val) tot_couples_train.append(couples_train) tot_couples_test.append(couples_test) # Train model on specific input set model = create_model(x_train, n_layers, neurons_per_layer) model.compile(optimizer='adam', loss='mean_squared_error', metrics=['mean_squared_error']) history = model.fit(x_train, y_train, epochs=epochs, shuffle=True, validation_data=(x_val, y_val), batch_size=batch_size, callbacks=[early_stopping, reduce_lr], verbose=False) # test it on average err among y_pred = np.array(model.predict(x_test)) mae_is0_avg.append(metrics.mae(y_avg_test, y_pred)) tmp = metrics.const_violation(x_test, y_pred) inversion_is0_avg.append(tmp[0]) plateau_is0_avg.append(tmp[1]) dist_const_is0_avg.append(metrics.error_average_distance(y_pred)) # test it on err of same input set as training y_pred = np.array(model.predict(x_test)) mae_is0_is0.append(metrics.mae(y_test, y_pred)) inversion_is0_is0.append(tmp[0]) plateau_is0_is0.append(tmp[1]) dist_const_is0_is0.append(metrics.error_average_distance(y_pred)) # Train model on avg error among input set model = create_model(x_train, n_layers, neurons_per_layer) model.compile(optimizer='adam', loss='mean_squared_error', metrics=['mean_squared_error']) history = model.fit(x_train, y_avg_train, epochs=epochs, shuffle=True, validation_data=(x_val, y_val), batch_size=batch_size, callbacks=[early_stopping, reduce_lr], verbose=False) # test it on average error y_pred = np.array(model.predict(x_test)) mae_avg_avg.append(metrics.mae(y_avg_test, y_pred)) tmp = metrics.const_violation(x_test, y_pred) inversion_avg_avg.append(tmp[0]) plateau_avg_avg.append(tmp[1]) dist_const_avg_avg.append(metrics.error_average_distance(y_pred)) # test it on error of specific input set y_pred = np.array(model.predict(x_test)) mae_avg_is0.append(metrics.mae(y_test, y_pred)) inversion_avg_is0.append(tmp[0]) plateau_avg_is0.append(tmp[1]) dist_const_avg_is0.append(metrics.error_average_distance(y_pred)) return (mae_is0_is0, inversion_is0_is0, plateau_is0_is0, dist_const_is0_is0, mae_is0_avg, inversion_is0_avg, plateau_is0_avg, dist_const_is0_avg, mae_avg_is0, inversion_avg_is0, plateau_avg_is0, dist_const_avg_is0, mae_avg_avg, inversion_avg_avg, plateau_avg_avg, dist_const_avg_avg, tot_couples_train, tot_couples_test)
def main(args): meta_info = { "POLLUTION": [5, 50, 14], "HR": [32, 50, 13], "BATTERY": [20, 50, 3] } output_directory = "output/" horizon = 10 output_dim = 1 dataset_name = args.dataset save_model_file = args.save_model_file load_model_file = args.load_model_file lower_trial = args.lower_trial upper_trial = args.upper_trial learning_rate = args.learning_rate meta_learning_rate = args.meta_learning_rate adaptation_steps = args.adaptation_steps batch_size = args.batch_size model_name = args.model is_test = args.is_test patience_stopping = args.stopping_patience epochs = args.epochs noise_level = args.noise_level noise_type = args.noise_type assert model_name in ("FCN", "LSTM"), "Model was not correctly specified" assert dataset_name in ("POLLUTION", "HR", "BATTERY") window_size, task_size, input_dim = meta_info[dataset_name] grid = [0., noise_level] train_data = pickle.load( open( "../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) train_data_ML = pickle.load( open( "../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) validation_data = pickle.load( open( "../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) validation_data_ML = pickle.load( open( "../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) test_data = pickle.load( open( "../../Data/TEST-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-NOML.pickle", "rb")) test_data_ML = pickle.load( open( "../../Data/TEST-" + dataset_name + "-W" + str(window_size) + "-T" + str(task_size) + "-ML.pickle", "rb")) results_dict = {} for trial in range(lower_trial, upper_trial): output_directory = "../../Models/" + dataset_name + "_" + model_name + "_MAML/" + str( trial) + "/" save_model_file_ = output_directory + save_model_file load_model_file_ = output_directory + load_model_file try: os.mkdir(output_directory) except OSError as error: print(error) f = open(output_directory + "/results3.txt", "a+") f.write("Learning rate :%f \n" % learning_rate) f.write("Meta-learning rate: %f \n" % meta_learning_rate) f.write("Adaptation steps: %f \n" % adaptation_steps) f.write("\n") f.close() if model_name == "LSTM": model = LSTMModel(batch_size=batch_size, seq_len=window_size, input_dim=input_dim, n_layers=2, hidden_dim=120, output_dim=1) elif model_name == "FCN": kernels = [8, 5, 3] if dataset_name != "POLLUTION" else [4, 2, 1] model = FCN(time_steps=window_size, channels=[input_dim, 128, 128, 128], kernels=kernels) model.cuda() maml = l2l.algorithms.MAML(model, lr=learning_rate, first_order=False) opt = optim.Adam(maml.parameters(), lr=meta_learning_rate) torch.backends.cudnn.enabled = False total_num_tasks = train_data_ML.x.shape[0] test(maml, model_name, dataset_name, test_data_ML, adaptation_steps, learning_rate) val_error = test(maml, model_name, dataset_name, validation_data_ML, adaptation_steps, learning_rate) early_stopping = EarlyStopping(patience=patience_stopping, model_file=save_model_file_, verbose=True) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=200, verbose=True) #early_stopping(val_error, maml) for iteration in range(epochs): # Creates a clone of model opt.zero_grad() iteration_error = 0.0 print(iteration) for task in range(batch_size): learner = maml.clone() task = np.random.randint(0, total_num_tasks - horizon) #task_qry = np.random.randint(1,horizon+1) x_spt, y_spt = train_data_ML[task] #x_qry, y_qry = train_data_ML[(task+1):(task+1+horizon)] x_qry, y_qry = train_data_ML[task + 1] x_qry = x_qry.reshape(-1, window_size, input_dim) y_qry = y_qry.reshape(-1, output_dim) if model_name == "FCN": x_qry = np.transpose(x_qry, [0, 2, 1]) x_spt = np.transpose(x_spt, [0, 2, 1]) x_spt, y_spt = to_torch(x_spt), to_torch(y_spt) x_qry = to_torch(x_qry) y_qry = to_torch(y_qry) #data augmentation epsilon = grid[np.random.randint(0, len(grid))] if noise_type == "additive": y_spt = y_spt + epsilon y_qry = y_qry + epsilon else: y_spt = y_spt * (1 + epsilon) y_qry = y_qry * (1 + epsilon) # Fast adapt for step in range(adaptation_steps): pred = learner(x_spt) error = mae(pred, y_spt) learner.adapt( error) #, allow_unused=True)#, allow_nograd=True) pred = learner(x_qry) evaluation_error = mae(pred, y_qry) iteration_error += evaluation_error # Meta-update the model parameters iteration_error /= batch_size iteration_error.backward() #retain_graph = True) print("loss iteration:", iteration_error.data) opt.step() if iteration % 1 == 0: val_error = test(maml, model_name, dataset_name, validation_data_ML, adaptation_steps, learning_rate) test_error = test(maml, model_name, dataset_name, test_data_ML, adaptation_steps, learning_rate) scheduler.step(val_error) print("Val error:", val_error) print("Test error:", test_error) early_stopping(val_error, maml) if early_stopping.early_stop: print("Early stopping") break maml.load_state_dict(torch.load(save_model_file_)) validation_error = test(maml, model_name, dataset_name, validation_data_ML, adaptation_steps, learning_rate) #validation_error2 = test(maml, model_name, dataset_name, validation_data_ML, adaptation_steps, learning_rate, with_early_stopping=True) #validation_error3 = test(maml, model_name, dataset_name, validation_data_ML, 10, learning_rate, with_early_stopping=True) #validation_error4 = test(maml, model_name, dataset_name, validation_data_ML, 10, learning_rate*0.1, with_early_stopping=True) test_error = test(maml, model_name, dataset_name, test_data_ML, adaptation_steps, learning_rate) #test_error2 = test(maml, model_name, dataset_name, test_data_ML, adaptation_steps , learning_rate, with_early_stopping=True) #test_error3 = test(maml, model_name, dataset_name, test_data_ML, 10 , learning_rate, with_early_stopping=True) #test_error4 = test(maml, model_name, dataset_name, test_data_ML, 10, learning_rate*0.1, with_early_stopping=True) f = open(output_directory + "/results3.txt", "a+") f.write("Dataset :%s \n" % dataset_name) f.write("Test error: %f \n" % test_error) #f.write("Test error2: %f \n" % test_error2) #f.write("Test error3: %f \n" % test_error3) #f.write("Test error4: %f \n" % test_error4) f.write("Validation error: %f \n" % validation_error) #f.write("Validation error2: %f \n" %validation_error2) #f.write("Validation error3: %f \n" %validation_error3) #f.write("Validation error4: %f \n" %validation_error4) f.write("\n") f.close() results_dict[str(trial) + "_val"] = validation_error results_dict[str(trial) + "_test"] = test_error np.save( "npy_objects/run03_" + dataset_name + "_" + model_name + "_" + noise_type + "_" + str(noise_level * 100000) + ".npy", results_dict)
def calculate_train_and_forecast_metrics( self, train: pd.DataFrame, oos: pd.DataFrame, target_index: int, hps: dict, horizon: int, mae_rmse_ignore_when_actual_and_pred_are_zero: bool, mape_ignore_when_actual_is_zero: bool): train_dataset = TrainDataset(train_df=train, target_index=target_index, hyperparams=hps, horizon=horizon) train_loader = DataLoader(train_dataset, batch_size=1, num_workers=1) inputs, train_actual = next(iter(train_loader)) inputs = inputs.to(device=self.device) self.net = self.net.to(device=self.device) train_pred = self.net(inputs.float()) train_actual = train_actual[0, 0, :].cpu().numpy() train_pred = train_pred[0, 0, :].cpu().detach().numpy() forecast_actual = oos.iloc[:horizon, target_index].values forecast_pred = self.predict(train_df, target_index, hps, horizon) assert (train_actual.shape == train_pred.shape) assert (forecast_actual.shape == forecast_pred.shape) train_dict = { 'mae': metrics.mae(train_actual, train_pred, mae_rmse_ignore_when_actual_and_pred_are_zero), 'rmse': metrics.rmse(train_actual, train_pred, mae_rmse_ignore_when_actual_and_pred_are_zero), 'mape': metrics.mape(train_actual, train_pred, mape_ignore_when_actual_is_zero), 'presence_accuracy': metrics.presence_accuracy(train_actual, train_pred), 'peak_accuracy': metrics.peak_accuracy(train_actual, train_pred), 'total_volume': int(metrics.total_actual_volume(train_actual)), 'num_timestamps_predicted_on': int(train_pred.shape[0]) } forecast_dict = { 'mae': metrics.mae(forecast_actual, forecast_pred, mae_rmse_ignore_when_actual_and_pred_are_zero), 'rmse': metrics.rmse(forecast_actual, forecast_pred, mae_rmse_ignore_when_actual_and_pred_are_zero), 'mape': metrics.mape(forecast_actual, forecast_pred, mape_ignore_when_actual_is_zero), 'presence_accuracy': metrics.presence_accuracy(forecast_actual, forecast_pred), 'peak_accuracy': metrics.peak_accuracy(forecast_actual, forecast_pred), 'total_volume': int(metrics.total_actual_volume(forecast_actual)), 'num_time_stamps_predicted_on': int(forecast_pred.shape[0]) } train_metrics = pd.DataFrame.from_dict(train_dict, columns=[None], orient='index').iloc[:, 0].round(3) forecast_metrics = pd.DataFrame.from_dict( forecast_dict, columns=[None], orient='index').iloc[:, 0].round(3) return train_metrics, forecast_metrics
import numpy as np import pandas as pd import matplotlib.pyplot as plt from linearRegression.linearRegression import LinearRegression from metrics import rmse, mae N = 30 P = 5 X = pd.DataFrame(np.random.randn(N, P)) y = X[1] X[5] = X[1] LR = LinearRegression(True) LR.fit_vectorised(X, y, 10) y_hat = LR.predict(X) print('RMSE: ', rmse(y_hat, y)) print('MAE: ', mae(y_hat, y)) print("Values of thetas are:", LR.coef_)
def test_mae(self): real_mae = 0.3000 predicted_mae = metrics.mae(self.real_y, self.predicted_y) npt.assert_almost_equal(real_mae, predicted_mae, decimal=4)
days_train = 365 x_test = x_test[-days_test:, :, -30:] # y_test = y_test[-days_test:,:] x_train = x_train[-days_train:, :, -30:] y_train = y_train[-days_train:, :] model = model_builder(x_train, y_train, args) model.fit(x_train, y_train) pred = model.predict(x_test, y_test) results[str(fold_num + 2013) + '/' + str(fold_num + 14)] = [ metrics.crps(pred), metrics.nll(pred), metrics.mae(pred), metrics.rmse(pred), metrics.smape(pred), metrics.corr(pred), metrics.mb_log(pred), metrics.sdp(pred) ] tf.keras.backend.clear_session() results['Average'] = results.mean(1) results['Average'].loc['SDP'] = np.abs(results.loc['SDP'].values[-1]).mean() plt.plot(pred.index, pred['True'], color='black') plt.plot(pred.index, pred['Pred'], color='red') plt.fill_between(pred.index, pred['Pred'] - pred['Std'],