def step(self,
adapted_params_list,
val_tasks,
is_training,
additional_loss_term=None):
self._optimizer.zero_grad()
post_update_losses = []
for adapted_params, task in zip(adapted_params_list, val_tasks):
preds = self._model(task.x, params=adapted_params)
if ~is_training:
preds = torch.clamp(preds, 0, 1)
loss = mae(preds, task.y)
else:
loss = self._loss_func(preds, task.y)
post_update_losses.append(loss)
if additional_loss_term is None:
mean_loss = torch.mean(torch.stack(post_update_losses))
else:
mean_loss = torch.mean(
torch.stack(post_update_losses)) + additional_loss_term
if is_training:
mean_loss.backward()
self._optimizer.step()
return mean_loss
def test(sess, model, generator, result_file):
pointwise_results = []
pairwise_results = []
mae_results = []
for city in CITIES:
# Validate the model on the entire validation set
generator.load_test_set(sess, city)
pds = []
gts = []
factors = []
for _ in trange(generator.test_batches_per_epoch[city], desc=city):
batch_img, batch_label, batch_factor = generator.get_next(sess)
pd = sess.run(model.prob, feed_dict={model.x: batch_img})
pds.extend(pd.tolist())
gts.extend(batch_label.tolist())
factors.extend(batch_factor.tolist())
pds = np.asarray(pds)
gts = np.asarray(gts)
pointwise_results.append(metrics.pointwise(pds, gts))
pairwise_results.append(metrics.pairwise(pds, gts, factors))
mae_results.append(metrics.mae(pds, gts))
layout = '{:15} {:>10} {:>10} {:>10} {:>10}'
print(layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE'))
result_file.write(
layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE\n'))
print('-' * 59)
result_file.write('-' * 59 + '\n')
test_sizes = []
for city, pointwise, pairwise, mae in zip(CITIES, pointwise_results,
pairwise_results, mae_results):
print(
layout.format(city, generator.test_sizes[city],
'{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}'.format(mae)))
result_file.write(
layout.format(city, generator.test_sizes[city],
'{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}\n'.format(mae)))
test_sizes.append(generator.test_sizes[city])
test_sizes = np.asarray(test_sizes, dtype=np.int)
total = np.sum(test_sizes)
avg_pointwise = np.sum(np.asarray(pointwise_results) * test_sizes) / total
avg_pairwise = np.sum(np.asarray(pairwise_results) * test_sizes) / total
avg_mae = np.sum(np.asarray(mae_results) * test_sizes) / total
print('-' * 59)
result_file.write('-' * 59 + '\n')
print(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise), '{:.3f}'.format(avg_mae)))
result_file.write(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise),
'{:.3f}\n'.format(avg_mae)))
result_file.flush()
def update(self, test_predictions, val_predictions=None, year=None):
self.test_predictions = self.test_predictions.append(test_predictions)
try:
self.test_metrics[str(year + 2014) + '/' + str(year + 15)] = [
metrics.crps(test_predictions),
metrics.nll(test_predictions),
metrics.mae(test_predictions),
metrics.rmse(test_predictions),
metrics.smape(test_predictions),
metrics.corr(test_predictions),
np.ma.masked_invalid(metrics.mb_log(test_predictions)).mean(),
metrics.sdp(test_predictions)
]
except:
pass
if year == 3:
self.test_metrics['Average'] = self.test_metrics.mean(1)
self.test_metrics['Average'].loc['SDP'] = np.abs(
self.test_metrics.loc['SDP'].values[-1]).mean()
try:
self.val_predictions = self.val_predictions.append(val_predictions)
self.val_metrics[str(year + 2013) + '/' + str(year + 14)] = [
metrics.crps(val_predictions),
metrics.nll(val_predictions),
metrics.mae(val_predictions),
metrics.rmse(val_predictions),
metrics.smape(val_predictions),
metrics.corr(val_predictions),
metrics.mb_log(val_predictions).mean(),
metrics.sdp(val_predictions)
]
except:
pass
self.val_metrics['Average'] = self.val_metrics.mean(1)
self.val_metrics['Average'].loc['SDP'] = np.abs(
self.val_metrics.loc['SDP'].values[-1]).mean()
self.test_metrics['Average'] = self.test_metrics.mean(1)
self.test_metrics['Average'].loc['SDP'] = np.abs(
self.test_metrics.loc['SDP'].values[-1]).mean()
def on_epoch_end(self, epoch, logs={}):
train_predict = np.asarray(self.model.predict(self.train_x))
train_predictions_indices = [
example_pred_probs.tolist().index(max(example_pred_probs))
for example_pred_probs in train_predict
]
train_predictions = [
self.labels[prediction] for prediction in train_predictions_indices
]
train_targets = [self.labels[target] for target in self.train_y]
logs['macro_f1'] = macro_f1(train_targets, train_predictions)
logs['macro_recall'] = macro_recall(train_targets, train_predictions)
logs['mae'] = mae(train_targets, train_predictions)
logs['macro_averaged_mae'] = macro_averaged_mae(
train_targets, train_predictions)
val_data = self.validation_data[:self.num_inputs]
val_predict = np.asarray(self.model.predict(val_data))
predictions_indices = [
example_pred_probs.tolist().index(max(example_pred_probs))
for example_pred_probs in val_predict
]
predictions = [
self.labels[prediction] for prediction in predictions_indices
]
val_targ = flatten(self.validation_data[self.num_inputs])
targets = [self.labels[target] for target in val_targ]
logs['val_macro_f1'] = macro_f1(targets, predictions)
logs['val_macro_recall'] = macro_recall(targets, predictions)
logs['val_mae'] = mae(targets, predictions)
logs['val_macro_averaged_mae'] = macro_averaged_mae(
targets, predictions)
def main():
# dataset has format like [user_id, song_id, play_count]
file = 'train_triplets.txt'
print("Loading data...")
load_data(file)
print("Starting evaluation...")
calc_neighbours()
print("Finished evaluations.")
print_top_songs_for_user(1)
print("Starting cross validation...")
print("RMSE result: ", str(rmse(train_set, test_set)))
print("MAE result: ", str(mae(train_set, test_set)))
print("NDCG result: ", str(ndcg(train_set, test_set)))
def evaluate(self, observed, estimated, zone, res = 0.5):
'''
Returns evaluation between observed and estimated rainfall maps
by computing following metrics:
['BIAS', 'CORREALTION', 'Nash-Sutcliffe', 'RMSE', 'MAE',
'MEAN_OBS','MEAN_EST']
Inputs:
observed - 2D array. Observed rainfall map.
estimated - 2D array. Estimated rainfall map.
zone - (2,2) tuple. Evaluation study zone [km x km]
Optional:
res - scalar. Resolution for comparison, [km].
Outputs:
metrics - dictionary. Statistical metrics:
['bias', 'corr', 'nash', 'rmse', 'mae',
'mean_obs','mean_est']
'''
# We neglect area that is not estimated for comparison purpose.
estimated[estimated <= -999] = -999
observed[estimated <= -999] = -999
((x0, x1), (y0, y1)) = zone
# Cut the zone for evaluation
t1, t2, t3, t4 = int(y0/res), int(y1/res), int(x0/res), int(x1/res)
observed = observed[t1:t2, t3:t4]
estimated = estimated[t1:t2, t3:t4]
est = estimated[estimated<>-999].flatten()
obs = observed[observed<>-999].flatten()
stats = dict()
stats['bias'] = metrics.bias(obs, est)
stats['corr'] = metrics.corr(obs, est)
stats['nash'] = metrics.nash(obs, est)
stats['rmse'] = metrics.rmse(obs, est)
stats['mae'] = metrics.mae(obs, est)
stats['mean_obs'] = metrics.average(obs)
stats['mean_est'] = metrics.average(est)
# additional metrics can be added
##stats['likelihood'] = metrics.likelihood(obs, est)
##stats['mape'] = metrics.mape(obs, est)
##stats['mse'] = metrics.mse(obs, est)
##stats['mspe'] = metrics.mspe(obs, est)
##stats['rmspe'] = metrics.rmspe(obs, est)
return stats
def plot(listbox, event):
selected_files = [listbox.get(i) for i in listbox.curselection()]
max_loc = None
for name in selected_files:
data = np.genfromtxt(name, delimiter=',', skip_header=1)
y_pred = data[:, 1]
y_true = data[:, 0]
label = '{0} MAE:{1:.2f}'.format(
os.path.splitext(os.path.basename(name))[0],
metrics.mae(y_true, y_pred))
if max_loc is None:
max_loc = np.argmax(y_true)
plt.plot(y_true, label='True', color='black')
plt.plot(y_pred, label=label)
else:
offset = np.argmax(y_true) - max_loc
plt.plot(np.arange(len(y_pred)) - offset, y_pred, label=label)
plt.legend()
plt.grid()
plt.show()
def main(args):
meta_info = {
"POLLUTION": [5, 50, 14],
"HR": [32, 50, 13],
"BATTERY": [20, 50, 3]
}
output_directory = "output/"
verbose = True
batch_size = 64
freeze_model_flag = True
params = {'batch_size': batch_size, 'shuffle': True, 'num_workers': 0}
dataset_name = args.dataset
model_name = args.model
learning_rate = args.learning_rate
save_model_file = args.save_model_file
load_model_file = args.load_model_file
lower_trial = args.lower_trial
upper_trial = args.upper_trial
is_test = args.is_test
epochs = args.epochs
experiment_id = args.experiment_id
adaptation_steps = args.adaptation_steps
assert model_name in ("FCN", "LSTM"), "Model was not correctly specified"
assert dataset_name in ("POLLUTION", "HR", "BATTERY")
window_size, task_size, input_dim = meta_info[dataset_name]
batch_size = 64
train_data = pickle.load(
open(
"../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-NOML.pickle", "rb"))
train_data_ML = pickle.load(
open(
"../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-ML.pickle", "rb"))
validation_data = pickle.load(
open(
"../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" +
str(task_size) + "-NOML.pickle", "rb"))
validation_data_ML = pickle.load(
open(
"../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" +
str(task_size) + "-ML.pickle", "rb"))
test_data = pickle.load(
open(
"../../Data/TEST-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-NOML.pickle", "rb"))
test_data_ML = pickle.load(
open(
"../../Data/TEST-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-ML.pickle", "rb"))
if is_test == 0:
test_data = validation_data
train_idx, val_idx, test_idx = split_idx_50_50(
test_data.file_idx) if is_test else split_idx_50_50(
validation_data.file_idx)
n_domains_in_test = np.max(test_data.file_idx) + 1
test_loss_list = []
initial_test_loss_list = []
trials_loss_list = []
#trial = 0
for trial in range(lower_trial, upper_trial):
output_directory = "../../Models/" + dataset_name + "_" + model_name + "_MAML/" + str(
trial) + "/"
#save_model_file_ = output_directory + "encoder_"+save_model_file
#save_model_file_2 = output_directory + save_model_file
save_model_file_ = output_directory + experiment_id + "_encoder_model.pt"
save_model_file_2 = output_directory + experiment_id + "_model.pt"
load_model_file_ = output_directory + load_model_file
model = LSTMModel(batch_size=batch_size,
seq_len=window_size,
input_dim=input_dim,
n_layers=2,
hidden_dim=120,
output_dim=1)
model2 = nn.Linear(120, 1)
model.cuda()
model2.cuda()
maml = l2l.algorithms.MAML(model2, lr=learning_rate, first_order=False)
model.load_state_dict(torch.load(save_model_file_))
maml.load_state_dict(torch.load(save_model_file_2))
n_domains_in_test = np.max(test_data.file_idx) + 1
error_list = []
y_list = []
for domain in range(n_domains_in_test):
x_test = test_data.x
y_test = test_data.y
temp_train_data = SimpleDataset(
x=np.concatenate([
x_test[np.concatenate([train_idx[domain],
val_idx[domain]])][np.newaxis, :],
x_test[test_idx[domain]][np.newaxis, :]
]),
y=np.concatenate([
y_test[np.concatenate([train_idx[domain],
val_idx[domain]])][np.newaxis, :],
y_test[test_idx[domain]][np.newaxis, :]
]))
total_tasks_test = len(test_data_ML)
learner = maml.clone() # Creates a clone of model
learner.cuda()
accum_error = 0.0
accum_std = 0.0
count = 0.0
input_dim = test_data_ML.x.shape[-1]
window_size = test_data_ML.x.shape[-2]
output_dim = test_data_ML.y.shape[-1]
task = 0
model2 = nn.Linear(120, 1)
model2.load_state_dict(copy.deepcopy(maml.module.state_dict()))
model.cuda()
model2.cuda()
x_spt, y_spt = temp_train_data[task]
x_qry = temp_train_data.x[(task + 1)]
y_qry = temp_train_data.y[(task + 1)]
if model_name == "FCN":
x_qry = np.transpose(x_qry, [0, 2, 1])
x_spt = np.transpose(x_spt, [0, 2, 1])
x_spt, y_spt = to_torch(x_spt), to_torch(y_spt)
x_qry = to_torch(x_qry)
y_qry = to_torch(y_qry)
opt2 = optim.SGD(list(model2.parameters()), lr=learning_rate)
#learner.module.train()
size_back = 300
step_size = task_size * size_back
#model2.eval()
for step in range(adaptation_steps):
print(step)
#model2.train()
for idx in range(x_spt.shape[0] - task_size * size_back,
x_spt.shape[0], step_size):
pred = model2(model.encoder(x_spt[idx:idx + step_size]))
print(pred.shape)
print(step_size)
error = mae(pred, y_spt[idx:idx + step_size])
print(error)
opt2.zero_grad()
error.backward()
#learner.adapt(error)
opt2.step()
#model2.eval()
#learner.module.eval()
step = x_qry.shape[0] // 255
for idx in range(0, x_qry.shape[0], step):
pred = model2(model.encoder(x_qry[idx:idx + step]))
error = mae(pred, y_qry[idx:idx + step])
accum_error += error.data
accum_std += error.data**2
count += 1
error = accum_error / count
y_list.append(y_qry.cpu().numpy())
error_list.append(float(error.cpu().numpy()))
print(np.mean(error_list))
print(error_list)
trials_loss_list.append(np.mean(error_list))
print("mean:", np.mean(trials_loss_list))
print("std:", np.std(trials_loss_list))
import datetime
from data import load_data, get_data, get_song_sets
from model import learning
from metrics import rmse, mae, ndcg
max_users = 1000
iterations = 5
print("Data loading...")
load_data(max_users)
print("Finished.")
# Cross validation
for i in range(0, iterations):
print("Iteration", i + 1, "/", iterations)
print("Learning is in process...")
learning_set, testing_set = get_song_sets()
data = get_data()
start_time = time.time()
learning(data, learning_set, 100)
finish_time = time.time()
print("Learning finished. Time:", datetime.timedelta(seconds=finish_time - start_time))
print("RMSE:", rmse(data, testing_set))
print("MAE: ", mae(data, testing_set))
print("NDCG:", ndcg(data, testing_set))
print("=====")
def main(args):
meta_info = {
"POLLUTION": [5, 50, 14],
"HR": [32, 50, 13],
"BATTERY": [20, 50, 3]
}
output_directory = "output/"
verbose = True
batch_size = 64
freeze_model_flag = True
params = {'batch_size': batch_size, 'shuffle': True, 'num_workers': 0}
dataset_name = args.dataset
model_name = args.model
learning_rate = args.learning_rate
save_model_file = args.save_model_file
load_model_file = args.load_model_file
lower_trial = args.lower_trial
upper_trial = args.upper_trial
is_test = args.is_test
epochs = args.epochs
experiment_id = args.experiment_id
adaptation_steps = args.adaptation_steps
assert model_name in ("FCN", "LSTM"), "Model was not correctly specified"
assert dataset_name in ("POLLUTION", "HR", "BATTERY")
window_size, task_size, input_dim = meta_info[dataset_name]
batch_size = 64
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
loss_fn = mae
train_data = pickle.load(
open(
"../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-NOML.pickle", "rb"))
train_data_ML = pickle.load(
open(
"../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-ML.pickle", "rb"))
validation_data = pickle.load(
open(
"../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" +
str(task_size) + "-NOML.pickle", "rb"))
validation_data_ML = pickle.load(
open(
"../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" +
str(task_size) + "-ML.pickle", "rb"))
test_data = pickle.load(
open(
"../../Data/TEST-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-NOML.pickle", "rb"))
test_data_ML = pickle.load(
open(
"../../Data/TEST-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-ML.pickle", "rb"))
# paramters wto increase capactiy of the model
n_layers_task_net = 2
n_layers_task_encoder = 1
n_layers_task_decoder = 1
hidden_dim_task_net = 120
hidden_dim_encoder = 120
hidden_dim_decoder = 120
input_dim_task_net = input_dim
input_dim_task_encoder = input_dim + 1
output_dim_task_net = 1
output_dim_task_decoder = input_dim + 1
output_dim = 1
if is_test == 0:
test_data = validation_data
train_idx, val_idx, test_idx = split_idx_50_50(
test_data.file_idx) if is_test else split_idx_50_50(
validation_data.file_idx)
n_domains_in_test = np.max(test_data.file_idx) + 1
test_loss_list = []
initial_test_loss_list = []
trials_loss_list = []
modulate_task_net = True
#trial = 0
for trial in range(lower_trial, upper_trial):
output_directory = "../../Models/" + dataset_name + "_" + model_name + "_MMAML/" + str(
trial) + "/"
#save_model_file_ = output_directory + "encoder_"+save_model_file
#save_model_file_2 = output_directory + save_model_file
save_model_file_encoder = output_directory + experiment_id + "_encoder_model.pt"
save_model_file_ = output_directory + experiment_id + "_model.pt"
load_model_file_ = output_directory + load_model_file
##creating the network
task_net = LSTMModel(batch_size=batch_size,
seq_len=window_size,
input_dim=input_dim_task_net,
n_layers=n_layers_task_net,
hidden_dim=hidden_dim_task_net,
output_dim=output_dim_task_net)
task_encoder = LSTMModel(batch_size=batch_size,
seq_len=task_size,
input_dim=input_dim_task_encoder,
n_layers=n_layers_task_encoder,
hidden_dim=hidden_dim_encoder,
output_dim=1)
task_decoder = LSTMDecoder(batch_size=1,
n_layers=n_layers_task_decoder,
seq_len=task_size,
output_dim=output_dim_task_decoder,
hidden_dim=hidden_dim_encoder,
latent_dim=hidden_dim_decoder,
device=device)
lmbd = Lambda(hidden_dim_encoder, hidden_dim_task_net)
multimodal_learner = MultimodalLearner(task_net, task_encoder,
task_decoder, lmbd,
modulate_task_net)
multimodal_learner.to(device)
output_layer = nn.Linear(120, 1)
output_layer.to(device)
maml = l2l.algorithms.MAML(output_layer,
lr=learning_rate,
first_order=False)
multimodal_learner.load_state_dict(torch.load(save_model_file_encoder))
maml.load_state_dict(torch.load(save_model_file_))
n_domains_in_test = np.max(test_data.file_idx) + 1
error_list = []
y_list = []
for domain in range(n_domains_in_test):
print("Domain:", domain)
x_test = test_data.x
y_test = test_data.y
temp_train_data = SimpleDataset(
x=np.concatenate([
x_test[np.concatenate([train_idx[domain],
val_idx[domain]])][np.newaxis, :],
x_test[test_idx[domain]][np.newaxis, :]
]),
y=np.concatenate([
y_test[np.concatenate([train_idx[domain],
val_idx[domain]])][np.newaxis, :],
y_test[test_idx[domain]][np.newaxis, :]
]))
total_tasks_test = len(test_data_ML)
learner = maml.clone() # Creates a clone of model
learner.cuda()
accum_error = 0.0
accum_std = 0.0
count = 0.0
input_dim = test_data_ML.x.shape[-1]
window_size = test_data_ML.x.shape[-2]
output_dim = test_data_ML.y.shape[-1]
task_id = 0
#model2 = nn.Linear(120, 1)
#model2.load_state_dict(copy.deepcopy(maml.module.state_dict()))
#model.cuda()
#model2.cuda()
output_layer = nn.Linear(120, 1)
output_layer.load_state_dict(
copy.deepcopy(maml.module.state_dict()))
output_layer.to(device)
x_spt, y_spt = temp_train_data[task_id]
x_qry = temp_train_data.x[(task_id + 1)]
y_qry = temp_train_data.y[(task_id + 1)]
task = get_task_encoder_input(
SimpleDataset(x=x_spt[-50:][np.newaxis, :],
y=y_spt[-50:][np.newaxis, :]))
task = to_torch(task)
if model_name == "FCN":
x_qry = np.transpose(x_qry, [0, 2, 1])
x_spt = np.transpose(x_spt, [0, 2, 1])
x_spt, y_spt = to_torch(x_spt), to_torch(y_spt)
x_qry = to_torch(x_qry)
y_qry = to_torch(y_qry)
opt2 = optim.SGD(list(output_layer.parameters()), lr=learning_rate)
#learner.module.train()
size_back = 200
step_size = task_size * size_back
multimodal_learner.train()
#model2.eval()
for step in range(adaptation_steps):
step_size = 1
error_accum = 0
count = 0
#model2.train()
for idx in range(0, x_spt.shape[0], step_size):
x_spt_encoding, (vrae_loss, _, _) = multimodal_learner(
x_spt[idx:idx + step_size], task, output_encoding=True)
pred = output_layer(x_spt_encoding)
error_accum += mae(pred, y_spt[idx:idx + step_size])
count += 1
opt2.zero_grad()
error = error_accum / count
error.backward()
#learner.adapt(error)
opt2.step()
#model2.eval()
#learner.module.eval()
multimodal_learner.eval()
step = x_qry.shape[0] // 255
error_accum = 0
count = 0
for idx in range(0, x_qry.shape[0], step):
x_qry_encoding, (vrae_loss, _,
_) = multimodal_learner(x_qry[idx:idx + step],
task,
output_encoding=True)
pred = output_layer(x_qry_encoding)
error = mae(pred, y_qry[idx:idx + step])
accum_error += error.data
accum_std += error.data**2
count += 1
error = accum_error / count
y_list.append(y_qry.cpu().numpy())
error_list.append(float(error.cpu().numpy()))
print(np.mean(error_list))
print(error_list)
trials_loss_list.append(np.mean(error_list))
print("mean:", np.mean(trials_loss_list))
print("std:", np.std(trials_loss_list))
def main(_):
_, features, labels = load_data(
os.path.join(FLAGS.data_dir, FLAGS.dataset, 'train.h5'))
# Training
model = GaussianNB()
model.fit(features, labels)
# Evaluation
pointwise_results = []
pairwise_results = []
mae_results = []
test_sizes = []
for city in CITIES:
factors, features, labels = load_data(
os.path.join(FLAGS.data_dir, FLAGS.dataset,
'val_{}.h5'.format(city)))
pd_probs = model.predict_proba(features)
onehot_labels = to_onehot(labels)
pointwise_results.append(
metrics.pointwise(pds=pd_probs, gts=onehot_labels))
pairwise_results.append(
metrics.pairwise(pds=pd_probs, gts=onehot_labels, factors=factors))
mae_results.append(metrics.mae(pds=pd_probs, gts=onehot_labels))
test_sizes.append(len(labels))
result_file = open('result_{}_nb.txt'.format(FLAGS.dataset), 'w')
layout = '{:15} {:>10} {:>10} {:>10} {:>10}'
print(layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE'))
result_file.write(
layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE\n'))
print('-' * 59)
result_file.write('-' * 59 + '\n')
for city, pointwise, pairwise, mae, tsize in zip(CITIES, pointwise_results,
pairwise_results,
mae_results, test_sizes):
print(
layout.format(city, tsize, '{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}'.format(mae)))
result_file.write(
layout.format(city, tsize, '{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}\n'.format(mae)))
test_sizes = np.asarray(test_sizes, dtype=np.int)
total = np.sum(test_sizes)
avg_pointwise = np.sum(np.asarray(pointwise_results) * test_sizes) / total
avg_pairwise = np.sum(np.asarray(pairwise_results) * test_sizes) / total
avg_mae = np.sum(np.asarray(mae_results) * test_sizes) / total
print('-' * 59)
result_file.write('-' * 59 + '\n')
print(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise), '{:.3f}'.format(avg_mae)))
result_file.write(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise),
'{:.3f}\n'.format(avg_mae)))
result_file.close()
def test(loss_fn, maml, multimodal_model, task_data, dataset_name, data_ML, adaptation_steps, learning_rate, noise_level, noise_type, is_test = True, horizon = 10):
total_tasks = len(data_ML)
task_size = data_ML.x.shape[-3]
input_dim = data_ML.x.shape[-1]
window_size = data_ML.x.shape[-2]
output_dim = data_ML.y.shape[-1]
if is_test:
step = total_tasks//100
else:
step = 1
step = 1 if step == 0 else step
grid = [0., noise_level]
accum_error = 0.0
count = 1.0
for task_idx in range(0, (total_tasks-horizon-1), step):
temp_file_idx = data_ML.file_idx[task_idx:task_idx+horizon+1]
if(len(np.unique(temp_file_idx))>1):
continue
learner = maml.clone()
x_spt, y_spt = data_ML[task_idx]
x_qry = data_ML.x[(task_idx+1):(task_idx+1+horizon)].reshape(-1, window_size, input_dim)
y_qry = data_ML.y[(task_idx+1):(task_idx+1+horizon)].reshape(-1, output_dim)
task = task_data[task_idx:task_idx+1].cuda()
x_spt, y_spt = to_torch(x_spt), to_torch(y_spt)
x_qry = to_torch(x_qry)
y_qry = to_torch(y_qry)
epsilon = grid[np.random.randint(0,len(grid))]
if noise_type == "additive":
y_spt = y_spt+epsilon
y_qry = y_qry+epsilon
else:
y_spt = y_spt*(1+epsilon)
y_qry = y_qry*(1+epsilon)
for step in range(adaptation_steps):
x_encoding, _ = multimodal_model(x_spt, task, output_encoding=True)
pred = learner(x_encoding)
error = loss_fn(pred, y_spt)
learner.adapt(error)
x_encoding, _ = multimodal_model(x_qry, task, output_encoding=True)
y_pred = learner(x_encoding)
y_pred = torch.clamp(y_pred, 0, 1)
error = mae(y_pred, y_qry)
accum_error += error.data
count += 1
error = accum_error/count
return error.cpu().numpy()
if __name__ == "__main__":
# Matrix of movie ratings
train_data, test_data = get_movie_matrix()
# Get all user mean values
user_mean = get_user_mean(train_data)
start = time.time()
prediction_matrix = pd.DataFrame(index=train_data.index)
for name, data in train_data.iteritems():
prediction_matrix[name] = main(train_data, name, k)
# break
logging.info("Process done in: {0:.2f} seconds".format(time.time() -
start))
inter_columns = np.intersect1d(prediction_matrix.columns.values,
test_data.columns.values)
small_pred = prediction_matrix[inter_columns].dropna(how='all')
small_test = test_data.loc[:, inter_columns].dropna(how='all')
print("Test Matrix\n", small_test)
print("Predicted Matrix\n", small_pred.loc[small_test.index, :])
logging.info('\nMetric Calculations RMSE and MAE')
rmse_value = metrics.rmse(test_data, prediction_matrix)
print(f'RMSE:\t{rmse_value}')
mae_value = metrics.mae(test_data, prediction_matrix)
print(f'MAE:\t{mae_value}')
# Preprocessing
X = shuffled.iloc[:, :-1].squeeze()
y = (shuffled.iloc[:, -1:]).T.squeeze()
len_estate = len(y)
# Splitting data
X_train, y_train = X.loc[:split*len_estate], y.loc[:split*len_estate]
X_test, y_test = X.loc[split*len_estate+1:].reset_index(
drop=True), y.loc[split*len_estate+1:].reset_index(drop=True)
# Learning tree
print("Please wait for some time, it takes time, you can change max depth if it takes too long time.")
tree = DecisionTree(criterion="information_gain", max_depth=max_depth)
tree.fit(X_train, y_train)
tree.plot()
# Printing accuracies for different depths
for depth in range(2, max_depth+1):
y_hat = tree.predict(X_test, max_depth=depth)
print("Depth: ", depth)
print('\tRMSE: ', rmse(y_hat, y_test))
print('\tMAE: ', mae(y_hat, y_test))
# Decision Tree Regressor from Sci-kit learn
dt = DecisionTreeRegressor(random_state=0)
dt.fit(X_train, y_train)
y_hat = pd.Series(dt.predict(X_test))
print('Sklearn RMSE: ', rmse(y_hat, y_test))
print('Sklearn MAE: ', mae(y_hat, y_test))
def test2(maml, model, model_name, dataset_name, test_data_ML, adaptation_steps, learning_rate, noise_level, noise_type, is_test = True, horizon = 10):
total_tasks_test = len(test_data_ML)
error_list = []
learner = maml.clone() # Creates a clone of model
learner.cuda()
accum_error = 0.0
accum_std = 0.0
count = 0.0
grid = [0., noise_level]
input_dim = test_data_ML.x.shape[-1]
window_size = test_data_ML.x.shape[-2]
output_dim = test_data_ML.y.shape[-1]
if is_test:
step = total_tasks_test//100
else:
step = 1
step = 1 if step == 0 else step
for task in range(0, (total_tasks_test-horizon-1), step):
temp_file_idx = test_data_ML.file_idx[task:task+horizon+1]
if(len(np.unique(temp_file_idx))>1):
continue
if model_name == "LSTM":
model2 = LSTMModel( batch_size=None, seq_len = None, input_dim = input_dim, n_layers = 2, hidden_dim = 120, output_dim =1)
elif model_name == "LSTM_MRA":
model2 = LSTMModel_MRA( batch_size=None, seq_len = window_size, input_dim = input_dim, n_layers = 2, hidden_dim = 120, output_dim =1)
elif model_name == "FCN":
kernels = [8,5,3] if window_size != 5 else [4,2,1]
model2 = FCN(time_steps = window_size, channels=[input_dim, 128, 128, 128] , kernels=kernels)
#model2.cuda()
#model2.load_state_dict(copy.deepcopy(maml.module.state_dict()))
#opt2 = optim.Adam(model2.parameters(), lr=learning_rate)
learner = maml.clone()
x_spt, y_spt = test_data_ML[task]
x_qry = test_data_ML.x[(task+1):(task+1+horizon)].reshape(-1, window_size, input_dim)
y_qry = test_data_ML.y[(task+1):(task+1+horizon)].reshape(-1, output_dim)
#x_qry = test_data_ML.x[(task+1)].reshape(-1, window_size, input_dim)
#y_qry = test_data_ML.y[(task+1)].reshape(-1, output_dim)
if model_name == "FCN":
x_qry = np.transpose(x_qry, [0,2,1])
x_spt = np.transpose(x_spt, [0,2,1])
x_spt, y_spt = to_torch(x_spt), to_torch(y_spt)
x_qry = to_torch(x_qry)
y_qry = to_torch(y_qry)
epsilon = grid[np.random.randint(0,len(grid))]
if noise_type == "additive":
y_spt = y_spt+epsilon
y_qry = y_qry+epsilon
else:
y_spt = y_spt*(1+epsilon)
y_qry = y_qry*(1+epsilon)
#learner.module.train()
#model2.eval()
for step in range(adaptation_steps):
#model2.train()
pred = learner(model.encoder(x_spt))
error = mae(pred, y_spt)
#opt2.zero_grad()
#error.backward()
learner.adapt(error)
#opt2.step()
#model2.eval()
#learner.module.eval()
y_pred = learner(model.encoder(x_qry))
y_pred = torch.clamp(y_pred, 0, 1)
error = mae(y_pred, y_qry)
accum_error += error.data
accum_std += error.data**2
count += 1
error = accum_error/count
print("std:", accum_std/count)
return error
def benchmark_test(n_train, n_test, n_val, bm):
(mae_is0_is0, inversion_is0_is0, plateau_is0_is0,
dist_const_is0_is0) = [], [], [], []
(mae_is0_avg, inversion_is0_avg, plateau_is0_avg,
dist_const_is0_avg) = [], [], [], []
(mae_avg_is0, inversion_avg_is0, plateau_avg_is0,
dist_const_avg_is0) = [], [], [], []
(mae_avg_avg, inversion_avg_avg, plateau_avg_avg,
dist_const_avg_avg) = [], [], [], []
tot_couples_train, tot_couples_test = [], []
for i in range(niter):
print("Experiment n. {}".format(i))
# reading i-th test and trainig set
df_train, df_test, df_val = reader.read_replicable_dataset(
n_train, n_test, bm, i)
ninput = len(list(df_train.filter(regex='var_*')))
width = ninput * 10
# normalizing test and training set
scaler = preprocessing.MinMaxScaler()
label_target = suff_label_target + str(0)
if improved_dataset == 1:
add_features = ed.getAdditionalFeatures(bm)
for add_feat in add_features:
df_train['{}-{}'.format(
add_feat[0], add_feat[1])
] = df_train[add_feat[0]] - df_train[add_feat[1]]
df_test['{}-{}'.format(
add_feat[0], add_feat[1])
] = df_test[add_feat[0]] - df_test[add_feat[1]]
df_val['{}-{}'.format(
add_feat[0], add_feat[1])
] = df_val[add_feat[0]] - df_val[add_feat[1]]
ninput += len(add_features)
#Arrange the df with err column at the end
for _label_target in label_targets:
df_temp = df_train.pop(_label_target)
df_train[_label_target] = df_temp
df_temp = df_test.pop(_label_target)
df_test[_label_target] = df_temp
df_temp = df_val.pop(_label_target)
df_val[_label_target] = df_temp
if cap_error:
# capping error
for _label_target in label_targets:
df_train[_label_target] = [cap if x > cap else x for x in
df_train[_label_target]]
df_test[_label_target] = [cap if x > cap else x for x in
df_test[_label_target]]
df_val[_label_target] = [cap if x > cap else x for x in
df_val[_label_target]]
# -log10(err)
for _label_target in label_targets:
df_train[_label_target] = [sys.float_info.min if 0 == x else
-np.log10(x) for x in df_train[_label_target]]
df_test[_label_target] = [sys.float_info.min if 0 == x else
-np.log10(x) for x in df_test[_label_target]]
df_val[_label_target] = [sys.float_info.min if 0 == x else
-np.log10(x) for x in df_val[_label_target]]
y_avg_train = np.mean(np.array(df_train[label_targets]), axis=1,
dtype='float32').reshape((-1, 1))
y_avg_test = np.mean(np.array(df_test[label_targets]), axis=1,
dtype='float32').reshape((-1, 1))
y_avg_val = np.mean(np.array(df_val[label_targets]), axis=1,
dtype='float32').reshape((-1, 1))
# splitting training and test set in features and targets
x_train = scaler.fit_transform(df_train.iloc[:,0:ninput])
y_train = scaler.fit_transform(np.array(df_train[label_target]
).reshape((-1, 1)))
x_test = scaler.fit_transform(df_test.iloc[:,0:ninput])
y_test = scaler.fit_transform(np.array(df_test[label_target]
).reshape((-1, 1)))
x_val = scaler.fit_transform(df_val.iloc[:,0:ninput])
y_val = scaler.fit_transform(np.array(df_val[label_target]
).reshape((-1, 1)))
(_, _, _, couples_train) = reader.remove_const_violations(
x_train, y_train, y_avg_train)
(_, _, _, couples_test) = reader.remove_const_violations(
x_test, y_test, y_avg_test)
(_, _, _, couples_val) = reader.remove_const_violations(
x_val, y_val, y_avg_val)
tot_couples_train.append(couples_train)
tot_couples_test.append(couples_test)
# Train model on specific input set
automl = autosklearn.regression.AutoSklearnRegressor(
time_left_for_this_task=120, per_run_time_limit=30,)
automl.fit(x_train, y_train)
# test it on average err among
y_pred = np.array(automl.predict(x_test))
mae_is0_avg.append(metrics.mae(y_avg_test, y_pred))
tmp = metrics.const_violation(x_test, y_pred)
inversion_is0_avg.append(tmp[0])
plateau_is0_avg.append(tmp[1])
dist_const_is0_avg.append(metrics.error_average_distance(y_pred))
# test it on err of same input set as training
mae_is0_is0.append(metrics.mae(y_test, y_pred))
inversion_is0_is0.append(tmp[0])
plateau_is0_is0.append(tmp[1])
dist_const_is0_is0.append(metrics.error_average_distance(y_pred))
# Train model on avg error among input set
automl = autosklearn.regression.AutoSklearnRegressor(
time_left_for_this_task=120, per_run_time_limit=30,)
automl.fit(x_train, y_avg_train)
# test it on average error
y_pred = np.array(automl.predict(x_test))
mae_avg_avg.append(metrics.mae(y_avg_test, y_pred))
tmp = metrics.const_violation(x_test, y_pred)
inversion_avg_avg.append(tmp[0])
plateau_avg_avg.append(tmp[1])
dist_const_avg_avg.append(metrics.error_average_distance(y_pred))
# test it on error of specific input set
mae_avg_is0.append(metrics.mae(y_test, y_pred))
inversion_avg_is0.append(tmp[0])
plateau_avg_is0.append(tmp[1])
dist_const_avg_is0.append(metrics.error_average_distance(y_pred))
return (mae_is0_is0, inversion_is0_is0, plateau_is0_is0, dist_const_is0_is0,
mae_is0_avg, inversion_is0_avg, plateau_is0_avg, dist_const_is0_avg,
mae_avg_is0, inversion_avg_is0, plateau_avg_is0, dist_const_avg_is0,
mae_avg_avg, inversion_avg_avg, plateau_avg_avg, dist_const_avg_avg,
tot_couples_train, tot_couples_test)
def test2(maml,
model_name,
test_data_ML,
adaptation_steps,
learning_rate,
with_early_stopping=False,
horizon=10):
total_tasks_test = len(test_data_ML)
error_list = []
learner = maml.clone() # Creates a clone of model
learner.cuda()
accum_error = 0.0
count = 0
input_dim = test_data_ML.x.shape[-1]
window_size = test_data_ML.x.shape[-2]
output_dim = test_data_ML.y.shape[-1]
for task in range(0, (total_tasks_test - horizon - 1),
total_tasks_test // 100):
temp_file_idx = test_data_ML.file_idx[task:task + horizon + 1]
if (len(np.unique(temp_file_idx)) > 1):
continue
if model_name == "LSTM":
model2 = LSTMModel(batch_size=None,
seq_len=None,
input_dim=input_dim,
n_layers=2,
hidden_dim=120,
output_dim=1)
elif model_name == "FCN":
kernels = [8, 5, 3] if window_size != 5 else [4, 2, 1]
model2 = FCN(time_steps=window_size,
channels=[input_dim, 128, 128, 128],
kernels=kernels)
#model2.cuda()
#model2.load_state_dict(copy.deepcopy(maml.module.state_dict()))
#opt2 = optim.Adam(model2.parameters(), lr=learning_rate)
learner = maml.clone()
x_spt, y_spt = test_data_ML[task]
x_qry = test_data_ML.x[(task + 1):(task + 1 + horizon)].reshape(
-1, window_size, input_dim)
y_qry = test_data_ML.y[(task + 1):(task + 1 + horizon)].reshape(
-1, output_dim)
if model_name == "FCN":
x_qry = np.transpose(x_qry, [0, 2, 1])
x_spt = np.transpose(x_spt, [0, 2, 1])
x_spt, y_spt = to_torch(x_spt), to_torch(y_spt)
x_qry = to_torch(x_qry)
y_qry = to_torch(y_qry)
early_stopping = EarlyStopping(patience=2,
model_file="temp/temp_file.pt",
verbose=True)
#learner.module.train()
#model2.eval()
for step in range(adaptation_steps):
#model2.train()
pred = learner(x_spt)
error = mae(pred, y_spt)
#opt2.zero_grad()
#error.backward()
learner.adapt(error)
#opt2.step()
if with_early_stopping:
with torch.no_grad():
model2.load_state_dict(
copy.deepcopy(learner.module.state_dict()))
#model2.eval()
pred = model2(x_qry)
error = mae(pred, y_qry)
early_stopping(error, model2)
if early_stopping.early_stop:
print("Early stopping")
break
if with_early_stopping:
model2.load_state_dict(torch.load("temp/temp_file.pt"))
#model2.eval()
#learner.module.eval()
pred = learner(x_qry)
error = mae(pred, y_qry)
accum_error += error.data
count += 1
error = accum_error / count
return error
def benchmark_test(n_train, n_test, n_val, bm, n_layers, neurons_per_layer):
(mae_is0_is0, inversion_is0_is0, plateau_is0_is0,
dist_const_is0_is0) = [], [], [], []
(mae_is0_avg, inversion_is0_avg, plateau_is0_avg,
dist_const_is0_avg) = [], [], [], []
(mae_avg_is0, inversion_avg_is0, plateau_avg_is0,
dist_const_avg_is0) = [], [], [], []
(mae_avg_avg, inversion_avg_avg, plateau_avg_avg,
dist_const_avg_avg) = [], [], [], []
tot_couples_train, tot_couples_test = [], []
for i in range(niter):
print("Experiment n. {}".format(i))
# reading i-th test and trainig set
df_train, df_test, df_val = reader.read_replicable_dataset(
n_train, n_test, bm, i)
ninput = len(list(df_train.filter(regex='var_*')))
width = ninput * 10
# normalizing test and training set
scaler = preprocessing.MinMaxScaler()
label_target = suff_label_target + str(0)
if improved_dataset == 1:
add_features = ed.getAdditionalFeatures(bm)
for add_feat in add_features:
df_train['{}-{}'.format(
add_feat[0], add_feat[1]
)] = df_train[add_feat[0]] - df_train[add_feat[1]]
df_test['{}-{}'.format(
add_feat[0],
add_feat[1])] = df_test[add_feat[0]] - df_test[add_feat[1]]
df_val['{}-{}'.format(
add_feat[0],
add_feat[1])] = df_val[add_feat[0]] - df_val[add_feat[1]]
ninput += len(add_features)
#Arrange the df with err column at the end
for _label_target in label_targets:
df_temp = df_train.pop(_label_target)
df_train[_label_target] = df_temp
df_temp = df_test.pop(_label_target)
df_test[_label_target] = df_temp
df_temp = df_val.pop(_label_target)
df_val[_label_target] = df_temp
if cap_error:
# capping error
for _label_target in label_targets:
df_train[_label_target] = [
cap if x > cap else x for x in df_train[_label_target]
]
df_test[_label_target] = [
cap if x > cap else x for x in df_test[_label_target]
]
df_val[_label_target] = [
cap if x > cap else x for x in df_val[_label_target]
]
# -log10(err)
for _label_target in label_targets:
df_train[_label_target] = [
sys.float_info.min if 0 == x else -np.log10(x)
for x in df_train[_label_target]
]
df_test[_label_target] = [
sys.float_info.min if 0 == x else -np.log10(x)
for x in df_test[_label_target]
]
df_val[_label_target] = [
sys.float_info.min if 0 == x else -np.log10(x)
for x in df_val[_label_target]
]
print(df_train)
sys.exit()
y_avg_train = np.mean(np.array(df_train[label_targets]),
axis=1,
dtype='float32').reshape((-1, 1))
y_avg_test = np.mean(np.array(df_test[label_targets]),
axis=1,
dtype='float32').reshape((-1, 1))
y_avg_val = np.mean(np.array(df_val[label_targets]),
axis=1,
dtype='float32').reshape((-1, 1))
# splitting training and test set in features and targets
x_train = scaler.fit_transform(df_train.iloc[:, 0:ninput])
y_train = scaler.fit_transform(
np.array(df_train[label_target]).reshape((-1, 1)))
x_test = scaler.fit_transform(df_test.iloc[:, 0:ninput])
y_test = scaler.fit_transform(
np.array(df_test[label_target]).reshape((-1, 1)))
x_val = scaler.fit_transform(df_val.iloc[:, 0:ninput])
y_val = scaler.fit_transform(
np.array(df_val[label_target]).reshape((-1, 1)))
(_, _, _, couples_train) = reader.remove_const_violations(
x_train, y_train, y_avg_train)
(_, _, _, couples_test) = reader.remove_const_violations(
x_test, y_test, y_avg_test)
(_, _, _,
couples_val) = reader.remove_const_violations(x_val, y_val, y_avg_val)
tot_couples_train.append(couples_train)
tot_couples_test.append(couples_test)
# Train model on specific input set
model = create_model(x_train, n_layers, neurons_per_layer)
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['mean_squared_error'])
history = model.fit(x_train,
y_train,
epochs=epochs,
shuffle=True,
validation_data=(x_val, y_val),
batch_size=batch_size,
callbacks=[early_stopping, reduce_lr],
verbose=False)
# test it on average err among
y_pred = np.array(model.predict(x_test))
mae_is0_avg.append(metrics.mae(y_avg_test, y_pred))
tmp = metrics.const_violation(x_test, y_pred)
inversion_is0_avg.append(tmp[0])
plateau_is0_avg.append(tmp[1])
dist_const_is0_avg.append(metrics.error_average_distance(y_pred))
# test it on err of same input set as training
y_pred = np.array(model.predict(x_test))
mae_is0_is0.append(metrics.mae(y_test, y_pred))
inversion_is0_is0.append(tmp[0])
plateau_is0_is0.append(tmp[1])
dist_const_is0_is0.append(metrics.error_average_distance(y_pred))
# Train model on avg error among input set
model = create_model(x_train, n_layers, neurons_per_layer)
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['mean_squared_error'])
history = model.fit(x_train,
y_avg_train,
epochs=epochs,
shuffle=True,
validation_data=(x_val, y_val),
batch_size=batch_size,
callbacks=[early_stopping, reduce_lr],
verbose=False)
# test it on average error
y_pred = np.array(model.predict(x_test))
mae_avg_avg.append(metrics.mae(y_avg_test, y_pred))
tmp = metrics.const_violation(x_test, y_pred)
inversion_avg_avg.append(tmp[0])
plateau_avg_avg.append(tmp[1])
dist_const_avg_avg.append(metrics.error_average_distance(y_pred))
# test it on error of specific input set
y_pred = np.array(model.predict(x_test))
mae_avg_is0.append(metrics.mae(y_test, y_pred))
inversion_avg_is0.append(tmp[0])
plateau_avg_is0.append(tmp[1])
dist_const_avg_is0.append(metrics.error_average_distance(y_pred))
return (mae_is0_is0, inversion_is0_is0, plateau_is0_is0,
dist_const_is0_is0, mae_is0_avg, inversion_is0_avg,
plateau_is0_avg, dist_const_is0_avg, mae_avg_is0,
inversion_avg_is0, plateau_avg_is0, dist_const_avg_is0,
mae_avg_avg, inversion_avg_avg, plateau_avg_avg,
dist_const_avg_avg, tot_couples_train, tot_couples_test)
def main(args):
meta_info = {
"POLLUTION": [5, 50, 14],
"HR": [32, 50, 13],
"BATTERY": [20, 50, 3]
}
output_directory = "output/"
horizon = 10
output_dim = 1
dataset_name = args.dataset
save_model_file = args.save_model_file
load_model_file = args.load_model_file
lower_trial = args.lower_trial
upper_trial = args.upper_trial
learning_rate = args.learning_rate
meta_learning_rate = args.meta_learning_rate
adaptation_steps = args.adaptation_steps
batch_size = args.batch_size
model_name = args.model
is_test = args.is_test
patience_stopping = args.stopping_patience
epochs = args.epochs
noise_level = args.noise_level
noise_type = args.noise_type
assert model_name in ("FCN", "LSTM"), "Model was not correctly specified"
assert dataset_name in ("POLLUTION", "HR", "BATTERY")
window_size, task_size, input_dim = meta_info[dataset_name]
grid = [0., noise_level]
train_data = pickle.load(
open(
"../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-NOML.pickle", "rb"))
train_data_ML = pickle.load(
open(
"../../Data/TRAIN-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-ML.pickle", "rb"))
validation_data = pickle.load(
open(
"../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" +
str(task_size) + "-NOML.pickle", "rb"))
validation_data_ML = pickle.load(
open(
"../../Data/VAL-" + dataset_name + "-W" + str(window_size) + "-T" +
str(task_size) + "-ML.pickle", "rb"))
test_data = pickle.load(
open(
"../../Data/TEST-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-NOML.pickle", "rb"))
test_data_ML = pickle.load(
open(
"../../Data/TEST-" + dataset_name + "-W" + str(window_size) +
"-T" + str(task_size) + "-ML.pickle", "rb"))
results_dict = {}
for trial in range(lower_trial, upper_trial):
output_directory = "../../Models/" + dataset_name + "_" + model_name + "_MAML/" + str(
trial) + "/"
save_model_file_ = output_directory + save_model_file
load_model_file_ = output_directory + load_model_file
try:
os.mkdir(output_directory)
except OSError as error:
print(error)
f = open(output_directory + "/results3.txt", "a+")
f.write("Learning rate :%f \n" % learning_rate)
f.write("Meta-learning rate: %f \n" % meta_learning_rate)
f.write("Adaptation steps: %f \n" % adaptation_steps)
f.write("\n")
f.close()
if model_name == "LSTM":
model = LSTMModel(batch_size=batch_size,
seq_len=window_size,
input_dim=input_dim,
n_layers=2,
hidden_dim=120,
output_dim=1)
elif model_name == "FCN":
kernels = [8, 5, 3] if dataset_name != "POLLUTION" else [4, 2, 1]
model = FCN(time_steps=window_size,
channels=[input_dim, 128, 128, 128],
kernels=kernels)
model.cuda()
maml = l2l.algorithms.MAML(model, lr=learning_rate, first_order=False)
opt = optim.Adam(maml.parameters(), lr=meta_learning_rate)
torch.backends.cudnn.enabled = False
total_num_tasks = train_data_ML.x.shape[0]
test(maml, model_name, dataset_name, test_data_ML, adaptation_steps,
learning_rate)
val_error = test(maml, model_name, dataset_name, validation_data_ML,
adaptation_steps, learning_rate)
early_stopping = EarlyStopping(patience=patience_stopping,
model_file=save_model_file_,
verbose=True)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt,
patience=200,
verbose=True)
#early_stopping(val_error, maml)
for iteration in range(epochs):
# Creates a clone of model
opt.zero_grad()
iteration_error = 0.0
print(iteration)
for task in range(batch_size):
learner = maml.clone()
task = np.random.randint(0, total_num_tasks - horizon)
#task_qry = np.random.randint(1,horizon+1)
x_spt, y_spt = train_data_ML[task]
#x_qry, y_qry = train_data_ML[(task+1):(task+1+horizon)]
x_qry, y_qry = train_data_ML[task + 1]
x_qry = x_qry.reshape(-1, window_size, input_dim)
y_qry = y_qry.reshape(-1, output_dim)
if model_name == "FCN":
x_qry = np.transpose(x_qry, [0, 2, 1])
x_spt = np.transpose(x_spt, [0, 2, 1])
x_spt, y_spt = to_torch(x_spt), to_torch(y_spt)
x_qry = to_torch(x_qry)
y_qry = to_torch(y_qry)
#data augmentation
epsilon = grid[np.random.randint(0, len(grid))]
if noise_type == "additive":
y_spt = y_spt + epsilon
y_qry = y_qry + epsilon
else:
y_spt = y_spt * (1 + epsilon)
y_qry = y_qry * (1 + epsilon)
# Fast adapt
for step in range(adaptation_steps):
pred = learner(x_spt)
error = mae(pred, y_spt)
learner.adapt(
error) #, allow_unused=True)#, allow_nograd=True)
pred = learner(x_qry)
evaluation_error = mae(pred, y_qry)
iteration_error += evaluation_error
# Meta-update the model parameters
iteration_error /= batch_size
iteration_error.backward() #retain_graph = True)
print("loss iteration:", iteration_error.data)
opt.step()
if iteration % 1 == 0:
val_error = test(maml, model_name, dataset_name,
validation_data_ML, adaptation_steps,
learning_rate)
test_error = test(maml, model_name, dataset_name, test_data_ML,
adaptation_steps, learning_rate)
scheduler.step(val_error)
print("Val error:", val_error)
print("Test error:", test_error)
early_stopping(val_error, maml)
if early_stopping.early_stop:
print("Early stopping")
break
maml.load_state_dict(torch.load(save_model_file_))
validation_error = test(maml, model_name, dataset_name,
validation_data_ML, adaptation_steps,
learning_rate)
#validation_error2 = test(maml, model_name, dataset_name, validation_data_ML, adaptation_steps, learning_rate, with_early_stopping=True)
#validation_error3 = test(maml, model_name, dataset_name, validation_data_ML, 10, learning_rate, with_early_stopping=True)
#validation_error4 = test(maml, model_name, dataset_name, validation_data_ML, 10, learning_rate*0.1, with_early_stopping=True)
test_error = test(maml, model_name, dataset_name, test_data_ML,
adaptation_steps, learning_rate)
#test_error2 = test(maml, model_name, dataset_name, test_data_ML, adaptation_steps , learning_rate, with_early_stopping=True)
#test_error3 = test(maml, model_name, dataset_name, test_data_ML, 10 , learning_rate, with_early_stopping=True)
#test_error4 = test(maml, model_name, dataset_name, test_data_ML, 10, learning_rate*0.1, with_early_stopping=True)
f = open(output_directory + "/results3.txt", "a+")
f.write("Dataset :%s \n" % dataset_name)
f.write("Test error: %f \n" % test_error)
#f.write("Test error2: %f \n" % test_error2)
#f.write("Test error3: %f \n" % test_error3)
#f.write("Test error4: %f \n" % test_error4)
f.write("Validation error: %f \n" % validation_error)
#f.write("Validation error2: %f \n" %validation_error2)
#f.write("Validation error3: %f \n" %validation_error3)
#f.write("Validation error4: %f \n" %validation_error4)
f.write("\n")
f.close()
results_dict[str(trial) + "_val"] = validation_error
results_dict[str(trial) + "_test"] = test_error
np.save(
"npy_objects/run03_" + dataset_name + "_" + model_name + "_" +
noise_type + "_" + str(noise_level * 100000) + ".npy", results_dict)
def calculate_train_and_forecast_metrics(
self, train: pd.DataFrame, oos: pd.DataFrame, target_index: int,
hps: dict, horizon: int,
mae_rmse_ignore_when_actual_and_pred_are_zero: bool,
mape_ignore_when_actual_is_zero: bool):
train_dataset = TrainDataset(train_df=train,
target_index=target_index,
hyperparams=hps,
horizon=horizon)
train_loader = DataLoader(train_dataset, batch_size=1, num_workers=1)
inputs, train_actual = next(iter(train_loader))
inputs = inputs.to(device=self.device)
self.net = self.net.to(device=self.device)
train_pred = self.net(inputs.float())
train_actual = train_actual[0, 0, :].cpu().numpy()
train_pred = train_pred[0, 0, :].cpu().detach().numpy()
forecast_actual = oos.iloc[:horizon, target_index].values
forecast_pred = self.predict(train_df, target_index, hps, horizon)
assert (train_actual.shape == train_pred.shape)
assert (forecast_actual.shape == forecast_pred.shape)
train_dict = {
'mae':
metrics.mae(train_actual, train_pred,
mae_rmse_ignore_when_actual_and_pred_are_zero),
'rmse':
metrics.rmse(train_actual, train_pred,
mae_rmse_ignore_when_actual_and_pred_are_zero),
'mape':
metrics.mape(train_actual, train_pred,
mape_ignore_when_actual_is_zero),
'presence_accuracy':
metrics.presence_accuracy(train_actual, train_pred),
'peak_accuracy':
metrics.peak_accuracy(train_actual, train_pred),
'total_volume':
int(metrics.total_actual_volume(train_actual)),
'num_timestamps_predicted_on':
int(train_pred.shape[0])
}
forecast_dict = {
'mae':
metrics.mae(forecast_actual, forecast_pred,
mae_rmse_ignore_when_actual_and_pred_are_zero),
'rmse':
metrics.rmse(forecast_actual, forecast_pred,
mae_rmse_ignore_when_actual_and_pred_are_zero),
'mape':
metrics.mape(forecast_actual, forecast_pred,
mape_ignore_when_actual_is_zero),
'presence_accuracy':
metrics.presence_accuracy(forecast_actual, forecast_pred),
'peak_accuracy':
metrics.peak_accuracy(forecast_actual, forecast_pred),
'total_volume':
int(metrics.total_actual_volume(forecast_actual)),
'num_time_stamps_predicted_on':
int(forecast_pred.shape[0])
}
train_metrics = pd.DataFrame.from_dict(train_dict,
columns=[None],
orient='index').iloc[:,
0].round(3)
forecast_metrics = pd.DataFrame.from_dict(
forecast_dict, columns=[None], orient='index').iloc[:, 0].round(3)
return train_metrics, forecast_metrics
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from linearRegression.linearRegression import LinearRegression
from metrics import rmse, mae
N = 30
P = 5
X = pd.DataFrame(np.random.randn(N, P))
y = X[1]
X[5] = X[1]
LR = LinearRegression(True)
LR.fit_vectorised(X, y, 10)
y_hat = LR.predict(X)
print('RMSE: ', rmse(y_hat, y))
print('MAE: ', mae(y_hat, y))
print("Values of thetas are:", LR.coef_)
def test_mae(self):
real_mae = 0.3000
predicted_mae = metrics.mae(self.real_y, self.predicted_y)
npt.assert_almost_equal(real_mae, predicted_mae, decimal=4)
days_train = 365
x_test = x_test[-days_test:, :, -30:]
# y_test = y_test[-days_test:,:]
x_train = x_train[-days_train:, :, -30:]
y_train = y_train[-days_train:, :]
model = model_builder(x_train, y_train, args)
model.fit(x_train, y_train)
pred = model.predict(x_test, y_test)
results[str(fold_num + 2013) + '/' + str(fold_num + 14)] = [
metrics.crps(pred),
metrics.nll(pred),
metrics.mae(pred),
metrics.rmse(pred),
metrics.smape(pred),
metrics.corr(pred),
metrics.mb_log(pred),
metrics.sdp(pred)
]
tf.keras.backend.clear_session()
results['Average'] = results.mean(1)
results['Average'].loc['SDP'] = np.abs(results.loc['SDP'].values[-1]).mean()
plt.plot(pred.index, pred['True'], color='black')
plt.plot(pred.index, pred['Pred'], color='red')
plt.fill_between(pred.index,
pred['Pred'] - pred['Std'],
