def test(self):
"""
Perform testing
"""
for i in range(self.n_loop):
# Test the model on the support set
for i_batch, (x1, x2, y, y0) in enumerate(self.user_data_loader):
x1, x2, y = x1.to(self.device), x2.to(self.device), y.to(
self.device)
# Predict the ratings on the support set
pred_y = self.model(x1, x2)
# Calculate the loss
loss = self.loss_fn(pred_y, y)
# Set the gradients to 0
self.optimizer.zero_grad()
# Update the local task-specific parameters
loss.backward()
# Perform gradient clipping
torch.nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
# Perform a parameter update based on the current gradient and the update rule
self.optimizer.step()
# Calculate the predicted ratings on the query set
q_pred_y = self.model(self.q_x1, self.q_x2)
# Calculate the MAE
mean_absolute_error = mae(self.q_y, q_pred_y)
# Calculate the NDCG
NDCG = ndcg(self.q_y, q_pred_y)
print("MAE: ", mean_absolute_error)
print("NDCG: ", NDCG)
return mean_absolute_error, NDCG
def evaluate(model, loader):
model.eval()
y_hat_list = []
y_list = []
for batch_data in loader:
a2a_g, b2a_g, b2b_gl, feats, types, counts, y = batch_data
_, y_hat = model(a2a_g, b2a_g, b2b_gl, types, counts)
y_hat_list += y_hat.tolist()
y_list += y.tolist()
y_hat = np.array(y_hat_list).reshape(-1,)
y = np.array(y_list).reshape(-1,)
return rmse(y, y_hat), mae(y, y_hat), sd(y, y_hat), pearson(y, y_hat)
def test():
print("------------test------------")
test_mae = 0
test_rmse = 0
net.eval()
with torch.no_grad():
for i, data in enumerate(test_loader):
# t = time()
x, e, y = data
output = net(x, adj, e)
# print(time() - t)
test_mae += utils.mae(output, y)
test_rmse += utils.rmse(output, y)
print("mae:{:2f} , rmse:{:2f}".format(test_mae / (i + 1),
test_rmse / (i + 1)))
masked_image = masked_image.cpu().numpy()
fake_image = fake_image.cpu().numpy()
image = image.cpu().numpy()
mask = mask.cpu().numpy()
image = scale(image, config.cgan_parameters["min"],
config.cgan_parameters["max"], -1, 1)
fake_image = scale(fake_image, config.cgan_parameters["min"],
config.cgan_parameters["max"], -1, 1)
masked_image = scale(masked_image, config.cgan_parameters["min"],
config.cgan_parameters["max"], -1, 1)
image = mask_lungs(image, mask)
fake_image = mask_lungs(fake_image, mask)
l1_diff = mae(image,
fake_image,
mask=mask,
mask_val=config.mask_values["non_lung_tissue"])
writer.add_scalar("L1 diff/Train", l1_diff, epoch)
f = create_figure([
masked_image[0, 0, :, :], fake_image[0, 0, :, :], image[0, 0, :, :]
],
figsize=(12, 4))
writer.add_figure("Image outputs/Real image, fake image, mask", f,
epoch)
log_images([masked_image, fake_image, image],
path=config.image_logs,
run_id=start_time,
step=epoch,
n_jobs=4, # -1 is ALL PROCESSOR AVAILABLE
cv=2, # None is K=5 fold CV
refit=True,
verbose=2)
# Fit the GridSearch
gcv.fit(X_train, y_train)
# Evaluate model
# train_preds = gcv.predict(X_test)
# train_preds = np.maximum(train_preds, 0) # Don't predict negative cases
# print('\nTrain MAE:', mae(train_preds, y_train))
test_preds = gcv.predict(X_test)
test_preds = np.maximum(test_preds, 0) # Don't predict negative cases
print('Test MAE:', mae(test_preds, y_test))
model_path = os.path.join(models_output_dir, model_name[:-2] + '.pkl')
print('Saving model in ', model_path)
logging.info('Saving model in ' + str(model_path))
# Save model to file
if not os.path.exists(models_output_dir):
os.mkdir(models_output_dir)
# Take a look at the best params
print(gcv.best_params_)
if model_name == 'LSTM()':
model_path = model_path.split('.')[0] + '.h5'
def train():
for epoch in range(epoches):
net.train()
train_count = 0
train_mae = 0
sum_train_loss = 0
with tqdm(total=len(train_loader),
desc='TRAINING-epoch-{}'.format(epoch),
unit='batches') as bar:
for i, data in enumerate(train_loader):
forward_t = time()
x, e, y = data
output = net(x, adj, e)
# output = net(x, adj)
backward_t = time()
loss = criterion(output, y)
sum_train_loss += loss.item()
# print("前向传播时间：", backward_t - forward_t)
opt.zero_grad()
loss.backward()
opt.step()
train_mae += utils.mae(output, y)
bar.set_postfix(loss=f'{sum_train_loss / (i + 1):.2f}',
mae=f'{train_mae / (i + 1):.2f}')
bar.update()
train_count += 1
# print("反向传播时间：", time() - backward_t)
# print("batch时间", time()-forward_t)
train_losses.append(sum_train_loss / train_count)
with torch.no_grad():
net.eval()
val_count = 0
val_mae = 0
sum_val_loss = 0
with tqdm(total=len(val_loader),
desc='VALIDATION-epoch-{}'.format(epoch),
unit='batches') as bar:
val_t = time()
for i, data in enumerate(val_loader):
x, e, y = data
output = net(x, adj, e)
# output = net(x, adj)
loss = criterion(output, y)
val_count += 1
val_mae += utils.mae(output, y)
sum_val_loss += loss.item()
bar.set_postfix(loss=f'{sum_val_loss / (i + 1):.2f}',
val_mae=f'{val_mae / (i + 1):.2f}')
bar.update()
# print("测试时间：", time()-val_t)
val_losses.append(sum_val_loss / val_count)
if (sum_val_loss / val_count) <= min(val_losses):
print("epoch-{:d} 保存模型。。。。。。".format(epoch))
torch.save(net,
f"experiment/{net_name}/net_{dataset}_{num_pred}.pkl")
test()
dict = {"train_loss": train_losses, "val_loss": val_losses}
return dict
def validate(val_loader, model, criterion, normalizer, test=False):
'''Test on validation/test set'''
model.eval()
# Statistics
batch_time = AverageMeter()
losses = AverageMeter()
if args.task == 'regression':
mae_errors = AverageMeter()
else:
accuracies = AverageMeter()
precisions = AverageMeter()
recalls = AverageMeter()
fscores = AverageMeter()
auc_scores = AverageMeter()
if test:
test_targets = []
test_preds = []
test_cif_ids = []
# Minibatches
end = time.time()
for i, (input, target, batch_cif_ids) in enumerate(val_loader):
# GPU
if args.cuda:
input_var = (Variable(input[0].cuda(async=True), volatile=True),
Variable(input[1].cuda(async=True),
volatile=True), input[2].cuda(async=True),
[crys_idx.cuda(async=True) for crys_idx in input[3]])
else:
input_var = (Variable(input[0], volatile=True),
Variable(input[1], volatile=True), input[2], input[3])
# Normalize
if args.task == 'regression':
target_normed = normalizer.norm(target)
else:
target_normed = target.view(-1).long()
if args.cuda:
target_var = Variable(target_normed.cuda(async=True),
volatile=True)
else:
target_var = Variable(target_normed, volatile=True)
# Forward
output = model(*input_var)
loss = criterion(output, target_var)
# Accuracy
if args.task == 'regression':
mae_error = mae(normalizer.denorm(output.data.cpu()), target)
losses.update(loss.data.cpu()[0], target.size(0))
mae_errors.update(mae_error, target.size(0))
if test:
test_pred = normalizer.denorm(output.data.cpu())
test_target = target
test_preds += test_pred.view(-1).tolist()
test_targets += test_target.view(-1).tolist()
test_cif_ids += batch_cif_ids
else:
accuracy, precision, recall, fscore, auc_score =\
class_eval(output.data.cpu(), target)
losses.update(loss.data.cpu()[0], target.size(0))
accuracies.update(accuracy, target.size(0))
precisions.update(precision, target.size(0))
recalls.update(recall, target.size(0))
fscores.update(fscore, target.size(0))
auc_scores.update(auc_score, target.size(0))
if test:
test_pred = torch.exp(output.data.cpu())
test_target = target
assert test_pred.shape[1] == 2
test_preds += test_pred[:, 1].tolist()
test_targets += test_target.view(-1).tolist()
test_cif_ids += batch_cif_ids
# Time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
if args.task == 'regression':
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'MAE {mae_errors.val:.3f} ({mae_errors.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
mae_errors=mae_errors))
else:
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Accu {accu.val:.3f} ({accu.avg:.3f})\t'
'Precision {prec.val:.3f} ({prec.avg:.3f})\t'
'Recall {recall.val:.3f} ({recall.avg:.3f})\t'
'F1 {f1.val:.3f} ({f1.avg:.3f})\t'
'AUC {auc.val:.3f} ({auc.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
accu=accuracies,
prec=precisions,
recall=recalls,
f1=fscores,
auc=auc_scores))
if test:
star_label = '**'
import csv
with open('test_results.csv', 'w') as f:
writer = csv.writer(f)
for cif_id, target, pred in zip(test_cif_ids, test_targets,
test_preds):
writer.writerow((cif_id, target, pred))
else:
star_label = '*'
if args.task == 'regression':
print(' {star} MAE {mae_errors.avg:.3f}'.format(star=star_label,
mae_errors=mae_errors))
return mae_errors.avg
else:
print(' {star} AUC {auc.avg:.3f}'.format(star=star_label,
auc=auc_scores))
return auc_scores.avg
def train(train_loader, model, criterion, optimizer, epoch, normalizer):
'''Train for a single epoch'''
model.train()
# Statistics
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
if args.task == 'regression':
mae_errors = AverageMeter()
else:
accuracies = AverageMeter()
precisions = AverageMeter()
recalls = AverageMeter()
fscores = AverageMeter()
auc_scores = AverageMeter()
# Loop through minibatches
end = time.time()
for i, (input, target, _) in enumerate(train_loader):
data_time.update(time.time() - end)
# GPU
if args.cuda:
input_var = (Variable(input[0].cuda(async=True)),
Variable(input[1].cuda(async=True)),
input[2].cuda(async=True),
[crys_idx.cuda(async=True) for crys_idx in input[3]])
else:
input_var = (Variable(input[0]), Variable(input[1]), input[2],
input[3])
# Normalize target
if args.task == 'regression':
target_normed = normalizer.norm(target)
else:
target_normed = target.view(-1).long()
if args.cuda:
target_var = Variable(target_normed.cuda(async=True))
else:
target_var = Variable(target_normed)
# Forward
output = model(*input_var)
loss = criterion(output, target_var)
# Backward
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Accuracy
if args.task == 'regression':
mae_error = mae(normalizer.denorm(output.data.cpu()), target)
losses.update(loss.data.cpu().item(), target.size(0))
mae_errors.update(mae_error, target.size(0))
else:
tmp = class_eval(output.data.cpu(), target)
accuracy, precision, recall, fscore, auc_score = tmp
losses.update(loss.data.cpu().item(), target.size(0))
accuracies.update(accuracy, target.size(0))
precisions.update(precision, target.size(0))
recalls.update(recall, target.size(0))
fscores.update(fscore, target.size(0))
auc_scores.update(auc_score, target.size(0))
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
if args.task == 'regression':
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'MAE {mae_errors.val:.3f} ({mae_errors.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
mae_errors=mae_errors))
else:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Accu {accu.val:.3f} ({accu.avg:.3f})\t'
'Precision {prec.val:.3f} ({prec.avg:.3f})\t'
'Recall {recall.val:.3f} ({recall.avg:.3f})\t'
'F1 {f1.val:.3f} ({f1.avg:.3f})\t'
'AUC {auc.val:.3f} ({auc.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
accu=accuracies,
prec=precisions,
recall=recalls,
f1=fscores,
auc=auc_scores))
test_data = pd.read_csv(cfg.vals['test_data'])
test_data = test_data[test_data.quantity > 0.0]
test_data_features = Features.feature_extraction(test_data, y_col='quantity')
X_test = test_data_features.toarray()
y_test = test_data['sales'].values
print("test data shape: {}".format(test_data.shape))
## Linear Regression
ols = LinearRegression(fit_intercept=True)
ols.fit(X_train, y_train)
y_hat = ols.predict(X_test)
test_data["y_hat"] = y_hat
test_mae = mae(y_hat, y_test)
test_rmse = rmse(y_hat, y_test)
test_mape = mape(y_hat, y_test)
print("--OLS--")
print("MAE - (test): {:.2f}".format(test_mae))
print("RMSE - (test): {:.2f}".format(test_rmse))
print("MAPE: - (test): {:.4f}".format(test_mape))
prod_errors = test_data[['region', 'time', 'sales', 'y_hat']].groupby(['time', "region"]).sum()
prod_mae = mae(prod_errors.y_hat, prod_errors.sales)
prod_rmse = rmse(prod_errors.y_hat, prod_errors.sales)
prod_mape = mape(prod_errors.y_hat, prod_errors.sales)
print("Region MAE - (test): {:.2f}".format(prod_mae))
print("Region RMSE - (test): {:.2f}".format(prod_rmse))
print("Region MAPE - (test): {:.4f}".format(prod_mape))
def make_report(self, report_name, id_test, x_test, y_test, country_test,
frame_test):
""" Runs evaluate on the provided data and generates a detailed error report """
if not os.path.exists('Reports/' + report_name):
os.mkdir('Reports/' + report_name)
results = self.predict(x_test)
# Generate detailied evaluation report
header = 'Country,Child,Frame'
for output_layer in self.get_config()['output_layers']:
header += ',{}_Actual'.format(output_layer[0])
for output_layer in self.get_config()['output_layers']:
header += ',{}_Prediction'.format(output_layer[0])
header += '\n'
with open('Reports/{}/evaluation_report.txt'.format(report_name),
'a') as f:
if os.stat('Reports/{}/evaluation_report.txt'.format(
report_name)).st_size == 0:
f.write(header)
for row in range(len(results)):
entry = ','.join([str(i) for i in country_test[row]]) + ','
entry += ','.join([str(i) for i in id_test[row]]) + ','
entry += ','.join([str(i) for i in frame_test[row]]) + ','
entry += ','.join([str(i) for i in y_test[row]]) + ','
entry += ','.join([str(i) for i in results[row]]) + '\n'
f.write(entry)
# Generate report of summary statistics
cultures = np.unique(country_test)
for c in cultures:
culture_rows = np.where(
country_test == c)[0] # get row numbers for culture c
culture_ids = id_test[culture_rows] # get ID rows for culture c
unique_ids = np.unique(culture_ids) # get unique IDs for culture c
for u in unique_ids:
all_id_rows = np.where(id_test == u)[0]
id_rows = np.intersect1d(
all_id_rows, culture_rows) # get ID rows for child u
id_icc = icc(results[id_rows],
y_test[id_rows])[0] # compute ICC for child u
id_pcc = pcc(results[id_rows],
y_test[id_rows])[0][0] # compute PCC for child u
id_ccc = ccc(results[id_rows],
y_test[id_rows]) # compute CCC for child u
id_mae = mae(results[id_rows],
y_test[id_rows]) # compute MAE for child u
icc_entry = '{},{},{}\n'.format(c, u, id_icc)
pcc_entry = '{},{},{}\n'.format(c, u, id_pcc)
ccc_entry = '{},{},{}\n'.format(c, u, id_ccc)
mae_entry = '{},{},{}\n'.format(c, u, id_mae)
with open('Reports/{}/icc_report.txt'.format(report_name),
'a') as f:
f.write(icc_entry)
with open('Reports/{}/pcc_report.txt'.format(report_name),
'a') as f:
f.write(pcc_entry)
with open('Reports/{}/ccc_report.txt'.format(report_name),
'a') as f:
f.write(ccc_entry)
with open('Reports/{}/mae_report.txt'.format(report_name),
'a') as f:
f.write(mae_entry)
return results
estimator=model,
param_grid=param_grid,
scoring=None, # TODO
n_jobs=1, # -1 is ALL PROCESSOR AVAILABLE
cv=2, # None is K=5 fold CV
refit=True,
)
# Fit the GridSearch
gcv.fit(X_samples, y_samples)
# Evaluate model
train_preds = gcv.predict(X_train)
train_preds = np.maximum(train_preds,
0) # Don't predict negative cases
print('\nTrain MAE:', mae(train_preds, y_train))
# test_preds = model.predict(X_test)
# test_preds = np.maximum(test_preds, 0) # Don't predict negative cases
# print('Test MAE:', mae(test_preds, y_test))
model_path = os.path.join(models_output_dir, model_name[:-2] + '.pkl')
print('Saving model in ', model_path)
logging.info('Saving model in ' + str(model_path))
# Save model to file
if not os.path.exists(models_output_dir):
os.mkdir(models_output_dir)
with open(model_path, 'wb') as model_file:
# --------------------- Compute decomposition --------------------------------------------
mytensor_training = mytensor[:, :, :num_days_train]
mytensor_valid = mytensor[:, :, num_days_train:num_days_train + num_days_valid]
mytensor_test = mytensor[:, :, num_days_train + num_days_valid:num_days]
stelar_model_predictions = None
best_model = None
best_score = float('Inf')
rank = [20]
mu = [1e-3]
nu = [1e-2]
for param in itertools.product(rank, nu, mu):
stelar_model = STELAR(rank=param[0],
nu=param[1],
mu=param[2],
max_iter=50,
inner_max_itr=50)
val_err = stelar_model.fit(mytensor_training, mytensor_valid)
if val_err < best_score:
best_score = val_err
best_model = stelar_model
stelar_model_predictions = best_model.predict(num_days)
rmse_stelar = rmse(stelar_model_predictions[:, 0, num_days - num_days_test:],
mytensor_test[:, 0, :])
mae_stelar = mae(stelar_model_predictions[:, 0, num_days - num_days_test:],
mytensor_test[:, 0, :])
print(f'RMSE STELAR : {rmse_stelar}, MAE STELAR : {mae_stelar}')