def critic_predict_dataflow(ctrl, data, log_dir, model_dir, vs_name):
"""
Prediction on a dataflow, used for testing a large batch of data
"""
ckpt = tf.train.latest_checkpoint(model_dir)
if not ckpt:
outputs = [0] * len(data[0])
logger.info("No model exists. Do not sort")
return outputs
model = critic_factory(ctrl, is_train=False, vs_name=vs_name)
ds_val = critic_dataflow_factory(ctrl, data, is_train=False)
output_names = ['{}/predicted_accuracy:0'.format(vs_name)]
session_config = None
if ctrl.critic_type == CriticTypes.LSTM:
session_config = tf.ConfigProto(device_count={'GPU': 0})
pred_config = PredictConfig(
model=model,
input_names=model.input_names,
output_names=output_names,
session_creator=NewSessionCreator(config=session_config),
session_init=SaverRestore(ckpt))
#with tf.Graph().as_default():
predictor = SimpleDatasetPredictor(pred_config, ds_val)
outputs = []
for o in predictor.get_result():
outputs.extend(o[0])
return outputs
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=[
'wrong-top1', 'wrong-top5', 'res-top5',
'label', 'logits'
])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
top5s = []
labels = []
logits = []
for top1, top5, pred, label, logit in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
top5s.extend(pred.tolist())
labels.extend(label.tolist())
logits.extend(logit.tolist())
with open("top5_resnet2x.json", "w") as f:
json.dump(top5s, f)
with open("labels_resnet2x.json", "w") as f:
json.dump(labels, f)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
return acc1.ratio, acc5.ratio
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5'])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5']
)
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_ILSVRC12(model, scale, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=[
'wrong-scale%03d-top1' % scale,
'wrong-scale%03d-top5' % scale
])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5 = RatioCounter(), RatioCounter()
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print('Top1/Top5 Acc: %.1f/%.1f' %
(100 - 100 * acc1.ratio, 100 - 100 * acc5.ratio))
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1',
'wrong-top5']) # 该函数用于构件图
pred = SimpleDatasetPredictor(
pred_config,
dataflow) # Simply create one predictor and run it on the DataFlow.
acc1, acc5 = RatioCounter(), RatioCounter(
) # A counter to count ratio of something.某事物的记录
for top1, top5 in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top5 Error: {}".format(acc5.ratio)) # 输出误差
def eval_on_ILSVRC12(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['wrong-top1', 'wrong-top5', 'attack_success'])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc5, succ = RatioCounter(), RatioCounter(), RatioCounter()
for top1, top5, num_succ in pred.get_result():
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc5.feed(top5.sum(), batch_size)
succ.feed(num_succ.sum(), batch_size)
# Uncomment to monitor the metrics during evaluation
# print("Top1 Error: {}".format(acc1.ratio))
# print("Attack Success Rate: {}".format(succ.ratio))
print("Top1 Error: {}".format(acc1.ratio))
print("Attack Success Rate: {}".format(succ.ratio))
print("Top5 Error: {}".format(acc5.ratio))
def eval_on_iNaturalist(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['label', 'logits', 'wrong-top1', 'wrong-top3'])
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc3 = RatioCounter(), RatioCounter()
for label, logits, top1, top3 in pred.get_result():
# from IPython import embed
# embed()
# pred_logits = logits[0]
# pred_logits = pred_logits.argsort()[-3:][::-1].tolist()
# print(pred_logits, label)
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc3.feed(top3.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top3 Error: {}".format(acc3.ratio))
def eval_on_iNaturalist(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['label','logits', 'wrong-top1', 'wrong-top3']
)
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1, acc3 = RatioCounter(), RatioCounter()
for label, logits, top1, top3 in pred.get_result():
# from IPython import embed
# embed()
# pred_logits = logits[0]
# pred_logits = pred_logits.argsort()[-3:][::-1].tolist()
# print(pred_logits, label)
batch_size = top1.shape[0]
acc1.feed(top1.sum(), batch_size)
acc3.feed(top3.sum(), batch_size)
print("Top1 Error: {}".format(acc1.ratio))
print("Top3 Error: {}".format(acc3.ratio))
def feature_extraction_on_AVA2012(model, sessinit, dataflow, data_format,
output_name, batch_size_times, feature_dir):
# set the configuration during the prediction process
# and apply the SimpleDatasetPredictor to extract the output_name
pred_config = PredictConfig(
model=model,
session_init=sessinit,
# NOTE: the names in input_names & output_names depends on the definitions in the loaded model
input_names=['input', 'label'],
output_names=[output_name])
pred = SimpleDatasetPredictor(pred_config, dataflow)
# BEGIN
cnt = 0
rep_dict = {output_name: []}
for outp in pred.get_result():
#
rep = outp[0]
if (len(rep.shape) == 4) and (data_format == 'NCHW'):
rep = np.transpose(rep, (0, 2, 3, 1))
rep_dict[output_name].append(rep)
#
cnt += 1
if cnt >= batch_size_times:
rep_dict[output_name] = np.concatenate(rep_dict[output_name],
axis=0)
print(' early stop for getting enough reps')
break
#
with open(
"{0}/reps_{1}.pkl".format(feature_dir,
output_name.replace('/', '-')),
'wb') as output_stream:
print('--> save {} to a local pkl file ...'.format(output_name))
print(' shape : ', end='')
print(rep_dict[output_name].shape)
pickle.dump(rep_dict, output_stream, protocol=2)
def test_on_iNaturalist(model, sessinit, dataflow):
pred_config = PredictConfig(
model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['label', 'logits']
)
pred = SimpleDatasetPredictor(pred_config, dataflow)
with open('./result_val.csv', 'w') as f:
writer = csv.writer(f)
writer.writerow(['id', 'predicted'])
for label, logits, in pred.get_result():
# from IPython import embed
# embed()
pred_logits = logits[0]
pred_logits = pred_logits.argsort()[-3:][::-1].tolist()
#print(label.tolist(), pred_logits)
row = []
# from IPython import embed
# embed()
row.append(label[0])
row.append(str(pred_logits[0]) + ' ' + str(pred_logits[1]) + ' ' + str(pred_logits[2]))
print(row)
writer.writerow(row)
choices=['label', 'label-prob', 'raw'])
parser.add_argument('--top', default=1, type=int)
args = parser.parse_args()
get_config_func = imp.load_source('config_script', args.config).get_config
# TODO not sure if it this script is still working
with tf.Graph().as_default() as G:
train_config = get_config_func()
config = PredictConfig(
inputs=train_config.inputs,
input_dataset_mapping=[train_config.inputs[0]], # assume first component is image
get_model_func=train_config.get_model_func,
session_init=sessinit.SaverRestore(args.model),
output_var_names=['output:0']
)
ds = ImageFromFile(args.images, 3, resize=(227, 227))
ds = BatchData(ds, 128, remainder=True)
predictor = SimpleDatasetPredictor(config, ds)
res = predictor.get_all_result()
if args.output_type == 'label':
for r in res:
print r[0].argsort(axis=1)[:,-args.top:][:,::-1]
elif args.output_type == 'label_prob':
raise NotImplementedError
elif args.output_type == 'raw':
print res
def viz_CAM(model,
sessinit,
name,
dataflow,
CAM_dir,
save_PKL=False,
save_REP=False):
# set the configuration during the prediction process
# and apply the SimpleDatasetPredictor to extract the output_names
pred_config = PredictConfig(
model=model,
session_init=sessinit,
# NOTE: the names in input_names & output_names depends on the definitions in the loaded model
input_names=['input', 'label'],
output_names=[
'wrong-top1', 'group3/block1/ReLU_output', 'linear_C2/W'
],
return_input=True)
pred = SimpleDatasetPredictor(pred_config, dataflow)
# create or clear CAM_dir for the output of results of CAM visualization
CAM_dir = '{}{}'.format(CAM_dir, name)
if os.path.isdir(CAM_dir):
print('--> clear the existing results in the directory {}'.format(
CAM_dir))
os.system('rm -r {}'.format(CAM_dir))
os.system('mkdir -p {}'.format(CAM_dir))
# for the sake of the ease of file government, we save
# jpgs, pkls and reps into three different directories
print('--> during the viz_CAM, we will generate the jpgs', end='')
os.system('mkdir -p {}'.format(CAM_dir + '/jpg'))
if save_PKL:
print(', pkl', end='')
os.system('mkdir -p {}'.format(CAM_dir + '/pkl'))
if save_REP:
print(', rep', end='')
os.system('mkdir -p {}'.format(CAM_dir + '/rep'))
print(' files for furthre usage')
# get the img_lab_list for proper formation of result recording
img_lab_list = dataset.AVA2012Meta().get_image_list(name)[0]
# BEGIN
cnt = 0
for inp, outp in pred.get_result():
#
images, labels = inp
wrongs, convmaps, W = outp
batch = wrongs.shape[0]
#
for i in range(batch):
convmap = convmaps[i, :, :, :] # 512 x 7 x 7
weight0 = W[:, 0].T # 512 x 1 for negative
mergedmap0_7x7 = np.matmul(weight0, convmap.reshape(
(512, -1))).reshape(7, 7)
mergedmap0 = cv2.resize(mergedmap0_7x7, (224, 224))
heatmap0 = viz.intensity_to_rgb(mergedmap0)
blend0 = images[i] * 0.5 + heatmap0 * 0.5
weight1 = W[:, 1].T # 512 x 1 for positive
mergedmap1_7x7 = np.matmul(weight1, convmap.reshape(
(512, -1))).reshape(7, 7)
mergedmap1 = cv2.resize(mergedmap1_7x7, (224, 224))
heatmap1 = viz.intensity_to_rgb(mergedmap1)
blend1 = images[i] * 0.5 + heatmap1 * 0.5
concat = np.concatenate(
(images[i], heatmap0, blend0, heatmap1, blend1), axis=1)
imgName, lab01 = img_lab_list[cnt]
assert lab01 == labels[i], \
'*** in viz_CAM: lab01 ({0}) != labels[i] ({1}) in image {2}'.format(lab01, labels[i], imgName)
# save image of CAM visualization
cv2.imwrite('{0}/jpg/cam_{1}_{2}_{3}.jpg'.format(CAM_dir, os.path.splitext(imgName)[0], \
lab01, int(wrongs[i])), concat)
# add @20171123: for CAMCrop
if save_PKL:
with open(
'{0}/pkl/{1}.pkl'.format(CAM_dir,
os.path.splitext(imgName)[0]),
'wb') as output_stream:
pickle.dump(
{
"GT01": lab01,
"CAM0": mergedmap0_7x7,
"CAM1": mergedmap1_7x7
}, output_stream)
if save_REP:
with open(
'{0}/rep/{1}.rep'.format(CAM_dir,
os.path.splitext(imgName)[0]),
'wb') as output_stream:
pickle.dump({
"convmap": convmap,
"W": W,
"GT01": lab01
}, output_stream)
cnt += 1
#
print(
'=== Finish CAM_viz on all the images in the validation dataset in AVA2012'
)
def eval_on_AVA2012(model,
sessinit,
dataflow,
repeat_times,
fusion_method='GlobalAverage',
output_predictions=True):
pred_config = PredictConfig(model=model,
session_init=sessinit,
input_names=['input', 'label'],
output_names=['softmax-logits', 'label'])
# add @ 20171127
def DeMaxMin_GlobalAverage(dps):
res01 = []
for cls in range(2):
s = dps[:, cls]
s_sum_denoise = np.sum(s) - np.max(s) - np.min(s)
res01.append(s_sum_denoise / (s.shape[0] - 2))
return res01
# add @ 20171127
def Median(dps):
res01 = []
for cls in range(2):
res01.append(np.median(dps[:, cls]))
return res01
#
def accuracyEstimation(log01_, GTs01_):
batch_size = log01_.shape[0]
acc01_ = np.zeros((int(batch_size // repeat_times), ))
y_True_Pred = np.zeros((int(batch_size // repeat_times), 2))
avg_p = np.zeros((int(batch_size // repeat_times), log01_.shape[1]))
#
for i in range(acc01_.shape[0]):
# change @ 20171127 : refine the fusion approaches
if fusion_method == 'GlobalAverage':
avgLog01__ = np.average(
log01_[(i * repeat_times):((i + 1) * repeat_times), :],
axis=0)
elif fusion_method == 'DeMaxMin_Average':
assert log01_.shape[0] >= 3, '*** ***'
avgLog01__ = DeMaxMin_GlobalAverage(
log01_[(i * repeat_times):((i + 1) * repeat_times), :])
elif fusion_method == 'Median':
avgLog01__ = Median(
log01_[(i * repeat_times):((i + 1) * repeat_times), :])
pred01__ = 0 if avgLog01__[0] > avgLog01__[
1] else 1 # TODO: confidence gap? or what if aesthetic_level > 2
# GTs01_ vs pred01__
acc01_[i] = int(pred01__ == GTs01_[(i * repeat_times)])
# add @ 20171122
y_True_Pred[i, 0] = GTs01_[(i * repeat_times)]
y_True_Pred[i, 1] = pred01__
# add @ 20171201
avg_p[i, :] = avgLog01__[:]
return acc01_, y_True_Pred, avg_p
pred = SimpleDatasetPredictor(pred_config, dataflow)
acc1 = RatioCounter()
y_True_Pred_list = []
interval_results = []
# add @ 20180709: to estimate the time consumption
elapsed_times = []
for pred_res in pred.get_result():
# for each image, we perform the prediction pipeline for repeat_times
# therefore, ...
logs01 = pred_res[0]
GTs01 = pred_res[1]
batch_size = logs01.shape[0]
assert batch_size % repeat_times == 0, \
'*** batch_size % repeat_times != 0, which makes the accuracyEstimation difficult ***'
start_time = time.time()
# change @ 20171122
# change @ 20171201
acc01, y_True_Pred_, avg_p = accuracyEstimation(logs01, GTs01)
elapsed_times.append(time.time() - start_time)
y_True_Pred_list.append(y_True_Pred_)
acc1.feed(acc01.sum(), acc01.shape[0])
# add @ 20171201
interval_results.append(np.hstack((y_True_Pred_, avg_p)))
# performance exhibition
print("--> detailed performance exhibition")
print(" Top1 Accuracy: {}".format(acc1.ratio))
# add @ 20171122
y_True_Pred_Matrix = np.vstack(y_True_Pred_list)
conf_matrix = confusion_matrix(y_True_Pred_Matrix[:, 0],
y_True_Pred_Matrix[:, 1])
print(" Confusion matrix is:")
print(conf_matrix)
print(" Accuracy of Negative Prediction: ",
(conf_matrix[0, 0]) / (conf_matrix[0, 0] + conf_matrix[1, 0]))
print(" Accuracy of Positive Prediction: ",
(conf_matrix[1, 1]) / (conf_matrix[0, 1] + conf_matrix[1, 1]))
print(" Recall of Negative Instances : ",
(conf_matrix[0, 0]) / (conf_matrix[0, 0] + conf_matrix[0, 1]))
print(" Recall of Positive Instances : ",
(conf_matrix[1, 1]) / (conf_matrix[1, 0] + conf_matrix[1, 1]))
# add @ 20171201
if output_predictions:
print(
' and save interval_results to ./interval_results_AVA2012.pkl for further investigation ...'
)
with open('./interval_results_AVA2012.pkl', 'wb') as output_stream:
pickle.dump({'interval_results': interval_results}, output_stream)
# add @ 20180709: exhibit the information of time consumption
print('--> average time consumption per image : {0:.3f}ms'.format( \
1000 * np.sum(elapsed_times) / y_True_Pred_Matrix.shape[0]))
choices=['label', 'label-prob', 'raw'])
parser.add_argument('--top', default=1, type=int)
args = parser.parse_args()
get_config_func = imp.load_source('config_script', args.config).get_config
# TODO not sure if it this script is still working
with tf.Graph().as_default() as G:
train_config = get_config_func()
config = PredictConfig(
inputs=train_config.inputs,
input_dataset_mapping=[train_config.inputs[0]
], # assume first component is image
get_model_func=train_config.get_model_func,
session_init=sessinit.SaverRestore(args.model),
output_var_names=['output:0'])
ds = ImageFromFile(args.images, 3, resize=(227, 227))
ds = BatchData(ds, 128, remainder=True)
predictor = SimpleDatasetPredictor(config, ds)
res = predictor.get_all_result()
if args.output_type == 'label':
for r in res:
print r[0].argsort(axis=1)[:, -args.top:][:, ::-1]
elif args.output_type == 'label_prob':
raise NotImplementedError
elif args.output_type == 'raw':
print res
