def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=[
'wrong-top1', 'wrong-top5', 'res-top5',
'label', 'logits'
])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
top5s = []
labels = []
logits = []
for top1, top5, pred, label, logit in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
top5s.extend(pred.tolist())
labels.extend(label.tolist())
logits.extend(logit.tolist())
with open("top5_resnet2x.json", "w") as f:
json.dump(top5s, f)
with open("labels_resnet2x.json", "w") as f:
json.dump(labels, f)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
return acc1.ratio, acc5.ratio
def eval_classification(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1','bn5/output:0','conv5/output:0']
)
acc1 = RatioCounter()
pred = FeedfreePredictor(pred_config, StagingInput(QueueInput(dataflow), device='/gpu:0'))
for _ in tqdm.trange(dataflow.size()):
top1,afbn5,beforbn5= pred()
dic ={}
dic['bn5/output:0']=afbn5
dic['conv5/output:0']=beforbn5
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
dir = logger.get_logger_dir()
fname = os.path.join(
dir, 'afbn5-{}.npz'.format(int(time.time())))
np.savez(fname, **dic)
print("Top1 Error: {}".format(acc1.ratio))
def eval_on_ILSVRC12(model_file, data_dir, outfile):
ds = get_data('val')
pred_config = PredictConfig(
model=Model(),
session_init=get_model_loader(model_file),
input_names=['input', 'label'],
output_names=['softmaxout', 'outputtop1', 'label']
# output_names=['wrong-top1', 'wrong-top5']
)
pred = SimpleDatasetPredictor(pred_config, ds)
acc1, acc5 = RatioCounter(), RatioCounter()
f = open(outfile, 'w')
for o in pred.get_result():
batch_size = o[0].shape[0]
# print batch_size
for i in range(batch_size):
confidence = o[1][i][0]
pred = np.where(o[0][i] == o[1][i])[0][0]
label = o[2][i]
# print confidence, pred, label
outline = "{} {} {}\n".format(confidence, pred, label)
f.write(outline)
# print o[1][i], np.where(o[0][i] == o[1][i]) , o[2][i]
# acc1.feed(o[0].sum(), batch_size)
# acc5.feed(o[1].sum(), batch_size)
# print("Top1 Error: {}".format(acc1.ratio))
# print("Top5 Error: {}".format(acc5.ratio))
f.close()
def eval(model_file, path, k, max_eval=None):
df_val = get_data(os.path.join(path, 'go_val.lmdb'),
shuffle=True,
isTrain=False)
if max_eval:
df_val = FixedSizeData(df_val, max_eval)
pred_config = PredictConfig(
model=Model(k, add_wrong=True),
session_init=get_model_loader(model_file),
input_names=['feature_planes', 'labels', 'labels_2d'],
output_names=['wrong-top1', 'wrong-top5'])
pred = SimpleDatasetPredictor(pred_config, df_val)
acc1, acc5 = RatioCounter(), RatioCounter()
try:
for o in pred.get_result():
batch_size = o[0].shape[0]
acc1.feed(o[0].sum(), batch_size)
acc5.feed(o[1].sum(), batch_size)
except Exception as e:
print e
from IPython import embed
embed()
err1 = (acc1.ratio) * 100
err5 = (acc5.ratio) * 100
print(
"Top1 Accuracy: {0:.2f}% Error: {1:.2f}% Random-Guess: ~0.44%".format(
100 - err1, err1))
print(
"Top5 Accuracy: {0:.2f}% Error: {1:.2f}% Random-Guess: ~2.00%".format(
100 - err5, err5))
def eval_on_cifar(model_file):
ds = get_data('test')
pred_config = PredictConfig(model=Model(n=NUM_UNITS),
session_init=get_model_loader(model_file),
input_names=['input', 'label'],
output_names=['incorrect_vector'])
pred = SimpleDatasetPredictor(pred_config, ds)
acc = RatioCounter()
for o in pred.get_result():
batch_size = o[0].shape[0]
acc.feed(o[0].sum(), batch_size)
print("Error: {}".format(acc.ratio))
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5'])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def run(model, sess_init):
pred_config = PredictConfig(model=model,
session_init=sess_init,
input_names=['input', 'label'],
output_names=['correct'])
dataset_train, dataset_test = get_data()
predictor = SimpleDatasetPredictor(pred_config, dataset_test)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1 in predictor.get_result():
batch_size = top1[0].shape[0]
acc1.feed(top1[0].sum(), batch_size)
print("Top1: {}".format(acc1.ratio))
def eval_on_ILSVRC12(model_file, data_dir):
ds = get_data('val')
pred_config = PredictConfig(model=Model(),
session_init=get_model_loader(model_file),
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5'])
pred = SimpleDatasetPredictor(pred_config, ds)
acc1, acc5 = RatioCounter(), RatioCounter()
for o in pred.get_result():
batch_size = o[0].shape[0]
acc1.feed(o[0].sum(), batch_size)
acc5.feed(o[1].sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model_file, data_dir, outfile, inputlist):
ds = get_data('val')
pred_config = PredictConfig(
model=Model(),
session_init=get_model_loader(model_file),
input_names=['input', 'label'],
output_names=['softmaxout', 'outputtop1', 'label', 'feat']
# output_names=['wrong-top1', 'wrong-top5']
)
pred = SimpleDatasetPredictor(pred_config, ds)
acc1, acc5 = RatioCounter(), RatioCounter()
lines = open(inputlist)
imglist = []
for line in lines:
imglist.append(line.strip().split()[0])
f = open(outfile, 'w')
count = 0
for o in pred.get_result():
batch_size = o[0].shape[0]
# print batch_size
for i in range(batch_size):
confidence = o[1][i][0]
pred = np.where(o[0][i] == o[1][i])[0][0]
f_feat_path = imglist[count] + '.bin'
f_path = imglist[count] + '.feat.bin'
count += 1
f_feat = open(f_feat_path, 'wb')
f_ = open(f_path, 'wb')
# print o[0][i]
# print o[0][i].shape
o[0][i].tofile(f_feat)
o[3][i].tofile(f_)
f_feat.close()
f_.close()
label = o[2][i]
# print confidence, pred, label
outline = "{} {} {}\n".format(confidence, pred, label)
f.write(outline)
# print o[1][i], np.where(o[0][i] == o[1][i]) , o[2][i]
# acc1.feed(o[0].sum(), batch_size)
# acc5.feed(o[1].sum(), batch_size)
# print("Top1 Error: {}".format(acc1.ratio))
# print("Top5 Error: {}".format(acc5.ratio))
f.close()
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5']
)
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model, scale, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=[
'wrong-scale%03d-top1' % scale,
'wrong-scale%03d-top5' % scale
])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print('Top1/Top5 Acc: %.1f/%.1f' %
(100 - 100 * acc1.ratio, 100 - 100 * acc5.ratio))
def eval_on_ILSVRC12(model_file, data_dir):
ds = get_data('val')
pred_config = PredictConfig(
model=Model(),
session_init=get_model_loader(model_file),
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5']
)
pred = SimpleDatasetPredictor(pred_config, ds)
acc1, acc5 = RatioCounter(), RatioCounter()
for o in pred.get_result():
batch_size = o[0].shape[0]
acc1.feed(o[0].sum(), batch_size)
acc5.feed(o[1].sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model_path, data_dir):
ds = dataset.ILSVRC12(data_dir, 'val', shuffle=False)
ds = AugmentImageComponent(ds, get_inference_augmentor())
ds = BatchData(ds, 192, remainder=True)
pred_config = PredictConfig(model=Model(),
session_init=get_model_loader(model_path),
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5'])
pred = SimpleDatasetPredictor(pred_config, ds)
acc1, acc5 = RatioCounter(), RatioCounter()
for o in pred.get_result():
batch_size = o[0].shape[0]
acc1.feed(o[0].sum(), batch_size)
acc5.feed(o[1].sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1',
'wrong-top5']) # 该函数用于构件图
pred = SimpleDatasetPredictor(
pred_config,
dataflow) # Simply create one predictor and run it on the DataFlow.
acc1, acc5 = RatioCounter(), RatioCounter(
) # A counter to count ratio of something.某事物的记录
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio)) # 输出误差
def eval_on_ILSVRC12(model_path, data_dir):
ds = dataset.ILSVRC12(data_dir, 'val', shuffle=False)
ds = AugmentImageComponent(ds, get_inference_augmentor())
ds = BatchData(ds, 192, remainder=True)
pred_config = PredictConfig(
model=Model(),
session_init=get_model_loader(model_path),
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5']
)
pred = SimpleDatasetPredictor(pred_config, ds)
acc1, acc5 = RatioCounter(), RatioCounter()
for o in pred.get_result():
batch_size = o[0].shape[0]
acc1.feed(o[0].sum(), batch_size)
acc5.feed(o[1].sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5'])
acc1, acc5 = RatioCounter(), RatioCounter()
# This does not have a visible improvement over naive predictor,
# but will have an improvement if image_dtype is set to float32.
pred = FeedfreePredictor(
pred_config, StagingInput(QueueInput(dataflow), device='/gpu:0'))
for _ in tqdm.trange(dataflow.size()):
top1, top5 = pred()
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_iNaturalist(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['label', 'logits', 'wrong-top1', 'wrong-top3'])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc3 = RatioCounter(), RatioCounter()
for label, logits, top1, top3 in pred.get_result():
# from IPython import embed
# embed()
# pred_logits = logits[0]
# pred_logits = pred_logits.argsort()[-3:][::-1].tolist()
# print(pred_logits, label)
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc3.feed(top3.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top3 Error: {}".format(acc3.ratio))
def test(net,
session_init,
val_dataflow,
do_calc_flops=False,
extended_log=False):
"""
Main test routine.
Parameters:
----------
net : obj
Model.
session_init : SessionInit
Session initializer.
do_calc_flops : bool, default False
Whether to calculate count of weights.
extended_log : bool, default False
Whether to log more precise accuracy values.
"""
pred_config = PredictConfig(
model=net,
session_init=session_init,
input_names=["input", "label"],
output_names=["wrong-top1", "wrong-top5"]
)
err_top1 = RatioCounter()
err_top5 = RatioCounter()
tic = time.time()
pred = FeedfreePredictor(pred_config, StagingInput(QueueInput(val_dataflow), device="/gpu:0"))
for _ in tqdm.trange(val_dataflow.size()):
err_top1_val, err_top5_val = pred()
batch_size = err_top1_val.shape[0]
err_top1.feed(err_top1_val.sum(), batch_size)
err_top5.feed(err_top5_val.sum(), batch_size)
err_top1_val = err_top1.ratio
err_top5_val = err_top5.ratio
if extended_log:
logging.info("Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})".format(
top1=err_top1_val, top5=err_top5_val))
else:
logging.info("Test: err-top1={top1:.4f}\terr-top5={top5:.4f}".format(
top1=err_top1_val, top5=err_top5_val))
logging.info("Time cost: {:.4f} sec".format(
time.time() - tic))
if do_calc_flops:
calc_flops(model=net)
def eval_on_iNaturalist(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['label','logits', 'wrong-top1', 'wrong-top3']
)
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc3 = RatioCounter(), RatioCounter()
for label, logits, top1, top3 in pred.get_result():
# from IPython import embed
# embed()
# pred_logits = logits[0]
# pred_logits = pred_logits.argsort()[-3:][::-1].tolist()
# print(pred_logits, label)
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc3.feed(top3.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top3 Error: {}".format(acc3.ratio))
class HorovodClassificationError(Inferencer):
"""
Like ClassificationError, it evaluates total samples & count of incorrect or correct samples.
But in the end we aggregate the total&count by horovod.
"""
def __init__(self, wrong_tensor_name, summary_name='validation_error'):
"""
Args:
wrong_tensor_name(str): name of the ``wrong`` binary vector tensor.
summary_name(str): the name to log the error with.
"""
self.wrong_tensor_name = wrong_tensor_name
self.summary_name = summary_name
def _setup_graph(self):
self._placeholder = tf.placeholder(tf.float32,
shape=[2],
name='to_be_reduced')
self._reduced = hvd.allreduce(self._placeholder, average=False)
def _before_inference(self):
self.err_stat = RatioCounter()
def _get_fetches(self):
return [self.wrong_tensor_name]
def _on_fetches(self, outputs):
vec = outputs[0]
batch_size = len(vec)
wrong = np.sum(vec)
self.err_stat.feed(wrong, batch_size)
# Uncomment this to monitor the metric during evaluation
# print(self.summary_name, self.err_stat.ratio)
def _after_inference(self):
tot = self.err_stat.total
cnt = self.err_stat.count
tot, cnt = self._reduced.eval(
feed_dict={self._placeholder: [tot, cnt]})
return {self.summary_name: cnt * 1. / tot}
def eval_classification(model, sessinit, dataflow):
"""
Eval a classification model on the dataset. It assumes the model inputs are
named "input" and "label", and contains "wrong-top1" and "wrong-top5" in the graph.
"""
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5'])
acc1, acc5 = RatioCounter(), RatioCounter()
# This does not have a visible improvement over naive predictor,
# but will have an improvement if image_dtype is set to float32.
pred = FeedfreePredictor(
pred_config, StagingInput(QueueInput(dataflow), device='/gpu:0'))
for _ in tqdm.trange(dataflow.size()):
top1, top5 = pred()
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def test(net,
session_init,
val_dataflow,
do_calc_flops=False,
extended_log=False):
pred_config = PredictConfig(
model=net,
session_init=session_init,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5']
)
err_top1 = RatioCounter()
err_top5 = RatioCounter()
tic = time.time()
pred = FeedfreePredictor(pred_config, StagingInput(QueueInput(val_dataflow), device='/gpu:0'))
# import tensorflow as tf
# summ_writer = tf.summary.FileWriter("/home/semery/projects/imgclsmob_data/gl-squeezenet_v1_1/", pred._sess.graph)
for _ in tqdm.trange(val_dataflow.size()):
err_top1_val, err_top5_val = pred()
batch_size = err_top1_val.shape[0]
err_top1.feed(err_top1_val.sum(), batch_size)
err_top5.feed(err_top5_val.sum(), batch_size)
# print("err_top1_val={}".format(err_top1_val.sum() / batch_size))
# print("err_top5_val={}".format(err_top5_val.sum() / batch_size))
err_top1_val = err_top1.ratio
err_top5_val = err_top5.ratio
if extended_log:
logging.info('Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})'.format(
top1=err_top1_val, top5=err_top5_val))
else:
logging.info('Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format(
top1=err_top1_val, top5=err_top5_val))
logging.info('Time cost: {:.4f} sec'.format(
time.time() - tic))
if do_calc_flops:
calc_flops(model=net)
def test(net,
session_init,
val_dataflow,
do_calc_flops=False,
extended_log=False):
pred_config = PredictConfig(
model=net,
session_init=session_init,
input_names=["input", "label"],
output_names=["wrong-top1", "wrong-top5"]
)
err_top1 = RatioCounter()
err_top5 = RatioCounter()
tic = time.time()
pred = FeedfreePredictor(pred_config, StagingInput(QueueInput(val_dataflow), device="/gpu:0"))
for _ in tqdm.trange(val_dataflow.size()):
err_top1_val, err_top5_val = pred()
batch_size = err_top1_val.shape[0]
err_top1.feed(err_top1_val.sum(), batch_size)
err_top5.feed(err_top5_val.sum(), batch_size)
err_top1_val = err_top1.ratio
err_top5_val = err_top5.ratio
if extended_log:
logging.info("Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})".format(
top1=err_top1_val, top5=err_top5_val))
else:
logging.info("Test: err-top1={top1:.4f}\terr-top5={top5:.4f}".format(
top1=err_top1_val, top5=err_top5_val))
logging.info("Time cost: {:.4f} sec".format(
time.time() - tic))
if do_calc_flops:
calc_flops(model=net)
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5', 'attack_success'])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5, succ = RatioCounter(), RatioCounter(), RatioCounter()
for top1, top5, num_succ in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
succ.feed(num_succ.sum(), batch_size)
# Uncomment to monitor the metrics during evaluation
# print("Top1 Error: {}".format(acc1.ratio))
# print("Attack Success Rate: {}".format(succ.ratio))
print("Top1 Error: {}".format(acc1.ratio))
print("Attack Success Rate: {}".format(succ.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def _before_inference(self):
self.err_stat = RatioCounter()
def eval_on_AVA2012(model,
sessinit,
dataflow,
repeat_times,
fusion_method='GlobalAverage',
output_predictions=True):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['softmax-logits', 'label'])
# add @ 20171127
def DeMaxMin_GlobalAverage(dps):
res01 = []
for cls in range(2):
s = dps[:, cls]
s_sum_denoise = np.sum(s) - np.max(s) - np.min(s)
res01.append(s_sum_denoise / (s.shape[0] - 2))
return res01
# add @ 20171127
def Median(dps):
res01 = []
for cls in range(2):
res01.append(np.median(dps[:, cls]))
return res01
#
def accuracyEstimation(log01_, GTs01_):
batch_size = log01_.shape[0]
acc01_ = np.zeros((int(batch_size // repeat_times), ))
y_True_Pred = np.zeros((int(batch_size // repeat_times), 2))
avg_p = np.zeros((int(batch_size // repeat_times), log01_.shape[1]))
#
for i in range(acc01_.shape[0]):
# change @ 20171127 : refine the fusion approaches
if fusion_method == 'GlobalAverage':
avgLog01__ = np.average(
log01_[(i * repeat_times):((i + 1) * repeat_times), :],
axis=0)
elif fusion_method == 'DeMaxMin_Average':
assert log01_.shape[0] >= 3, '*** ***'
avgLog01__ = DeMaxMin_GlobalAverage(
log01_[(i * repeat_times):((i + 1) * repeat_times), :])
elif fusion_method == 'Median':
avgLog01__ = Median(
log01_[(i * repeat_times):((i + 1) * repeat_times), :])
pred01__ = 0 if avgLog01__[0] > avgLog01__[
1] else 1 # TODO: confidence gap? or what if aesthetic_level > 2
# GTs01_ vs pred01__
acc01_[i] = int(pred01__ == GTs01_[(i * repeat_times)])
# add @ 20171122
y_True_Pred[i, 0] = GTs01_[(i * repeat_times)]
y_True_Pred[i, 1] = pred01__
# add @ 20171201
avg_p[i, :] = avgLog01__[:]
return acc01_, y_True_Pred, avg_p
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1 = RatioCounter()
y_True_Pred_list = []
interval_results = []
# add @ 20180709: to estimate the time consumption
elapsed_times = []
for pred_res in pred.get_result():
# for each image, we perform the prediction pipeline for repeat_times
# therefore, ...
logs01 = pred_res[0]
GTs01 = pred_res[1]
batch_size = logs01.shape[0]
assert batch_size % repeat_times == 0, \
'*** batch_size % repeat_times != 0, which makes the accuracyEstimation difficult ***'
start_time = time.time()
# change @ 20171122
# change @ 20171201
acc01, y_True_Pred_, avg_p = accuracyEstimation(logs01, GTs01)
elapsed_times.append(time.time() - start_time)
y_True_Pred_list.append(y_True_Pred_)
acc1.feed(acc01.sum(), acc01.shape[0])
# add @ 20171201
interval_results.append(np.hstack((y_True_Pred_, avg_p)))
# performance exhibition
print("--> detailed performance exhibition")
print(" Top1 Accuracy: {}".format(acc1.ratio))
# add @ 20171122
y_True_Pred_Matrix = np.vstack(y_True_Pred_list)
conf_matrix = confusion_matrix(y_True_Pred_Matrix[:, 0],
y_True_Pred_Matrix[:, 1])
print(" Confusion matrix is:")
print(conf_matrix)
print(" Accuracy of Negative Prediction: ",
(conf_matrix[0, 0]) / (conf_matrix[0, 0] + conf_matrix[1, 0]))
print(" Accuracy of Positive Prediction: ",
(conf_matrix[1, 1]) / (conf_matrix[0, 1] + conf_matrix[1, 1]))
print(" Recall of Negative Instances : ",
(conf_matrix[0, 0]) / (conf_matrix[0, 0] + conf_matrix[0, 1]))
print(" Recall of Positive Instances : ",
(conf_matrix[1, 1]) / (conf_matrix[1, 0] + conf_matrix[1, 1]))
# add @ 20171201
if output_predictions:
print(
' and save interval_results to ./interval_results_AVA2012.pkl for further investigation ...'
)
with open('./interval_results_AVA2012.pkl', 'wb') as output_stream:
pickle.dump({'interval_results': interval_results}, output_stream)
# add @ 20180709: exhibit the information of time consumption
print('--> average time consumption per image : {0:.3f}ms'.format( \
1000 * np.sum(elapsed_times) / y_True_Pred_Matrix.shape[0]))
