def test(sess, model, generator, result_file):
pointwise_results = []
pairwise_results = []
mae_results = []
for city in CITIES:
# Validate the model on the entire validation set
generator.load_test_set(sess, city)
pds = []
gts = []
factors = []
for _ in trange(generator.test_batches_per_epoch[city], desc=city):
batch_img, batch_label, batch_factor = generator.get_next(sess)
pd = sess.run(model.prob, feed_dict={model.x: batch_img})
pds.extend(pd.tolist())
gts.extend(batch_label.tolist())
factors.extend(batch_factor.tolist())
pds = np.asarray(pds)
gts = np.asarray(gts)
pointwise_results.append(metrics.pointwise(pds, gts))
pairwise_results.append(metrics.pairwise(pds, gts, factors))
mae_results.append(metrics.mae(pds, gts))
layout = '{:15} {:>10} {:>10} {:>10} {:>10}'
print(layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE'))
result_file.write(
layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE\n'))
print('-' * 59)
result_file.write('-' * 59 + '\n')
test_sizes = []
for city, pointwise, pairwise, mae in zip(CITIES, pointwise_results,
pairwise_results, mae_results):
print(
layout.format(city, generator.test_sizes[city],
'{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}'.format(mae)))
result_file.write(
layout.format(city, generator.test_sizes[city],
'{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}\n'.format(mae)))
test_sizes.append(generator.test_sizes[city])
test_sizes = np.asarray(test_sizes, dtype=np.int)
total = np.sum(test_sizes)
avg_pointwise = np.sum(np.asarray(pointwise_results) * test_sizes) / total
avg_pairwise = np.sum(np.asarray(pairwise_results) * test_sizes) / total
avg_mae = np.sum(np.asarray(mae_results) * test_sizes) / total
print('-' * 59)
result_file.write('-' * 59 + '\n')
print(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise), '{:.3f}'.format(avg_mae)))
result_file.write(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise),
'{:.3f}\n'.format(avg_mae)))
result_file.flush()
def main(_):
_, features, labels = load_data(
os.path.join(FLAGS.data_dir, FLAGS.dataset, 'train.h5'))
# Training
model = GaussianNB()
model.fit(features, labels)
# Evaluation
pointwise_results = []
pairwise_results = []
mae_results = []
test_sizes = []
for city in CITIES:
factors, features, labels = load_data(
os.path.join(FLAGS.data_dir, FLAGS.dataset,
'val_{}.h5'.format(city)))
pd_probs = model.predict_proba(features)
onehot_labels = to_onehot(labels)
pointwise_results.append(
metrics.pointwise(pds=pd_probs, gts=onehot_labels))
pairwise_results.append(
metrics.pairwise(pds=pd_probs, gts=onehot_labels, factors=factors))
mae_results.append(metrics.mae(pds=pd_probs, gts=onehot_labels))
test_sizes.append(len(labels))
result_file = open('result_{}_nb.txt'.format(FLAGS.dataset), 'w')
layout = '{:15} {:>10} {:>10} {:>10} {:>10}'
print(layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE'))
result_file.write(
layout.format('City', 'Size', 'Pointwise', 'Pairwise', 'MAE\n'))
print('-' * 59)
result_file.write('-' * 59 + '\n')
for city, pointwise, pairwise, mae, tsize in zip(CITIES, pointwise_results,
pairwise_results,
mae_results, test_sizes):
print(
layout.format(city, tsize, '{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}'.format(mae)))
result_file.write(
layout.format(city, tsize, '{:.3f}'.format(pointwise),
'{:.3f}'.format(pairwise), '{:.3f}\n'.format(mae)))
test_sizes = np.asarray(test_sizes, dtype=np.int)
total = np.sum(test_sizes)
avg_pointwise = np.sum(np.asarray(pointwise_results) * test_sizes) / total
avg_pairwise = np.sum(np.asarray(pairwise_results) * test_sizes) / total
avg_mae = np.sum(np.asarray(mae_results) * test_sizes) / total
print('-' * 59)
result_file.write('-' * 59 + '\n')
print(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise), '{:.3f}'.format(avg_mae)))
result_file.write(
layout.format('Avg.', total, '{:.3f}'.format(avg_pointwise),
'{:.3f}'.format(avg_pairwise),
'{:.3f}\n'.format(avg_mae)))
result_file.close()