def reduce_to_k_regions(k,rois,roi_scores, fc7,new_nms_thresh,score_threshold,minimal_surface):
""" Reduce the number of region to k or less
but it can even return more than k regions if we go out of the loop on new_nms_thresh
"""
if(len(fc7) <= k):
return(rois,roi_scores, fc7)
keep = np.where(roi_scores> score_threshold)
rois = rois[keep[0], :]
roi_scores = roi_scores[keep]
fc7 = fc7[keep[0],:]
if(len(fc7) <= k):
return(rois,roi_scores, fc7)
width = rois[:,2] - rois[:,0] +1
height = rois[:,3] - rois[:,1] +1
surface = width*height
keep = np.where(surface > minimal_surface)
rois = rois[keep[0], :]
roi_scores = roi_scores[keep]
fc7 = fc7[keep[0],:]
if(len(fc7) <= k):
return(rois,roi_scores, fc7)
#new_nms_thresh = 0.0
keep_all = []
for i in range(7):
rois_plus_scores = np.hstack((rois[:,1:5],roi_scores.reshape(-1,1)))
tmp_keep = nms(rois_plus_scores,new_nms_thresh)
keep_new = np.setdiff1d(tmp_keep,keep_all) # Nouveau index
keep_all2 = np.union1d(keep_all,tmp_keep) # sorted
if len(keep_all2) > k:
keep = np.union1d(keep_all,keep_new[0:k-len(keep_all)]).astype(int)
assert(len(keep)==k)
rois = rois[keep, :]
roi_scores = roi_scores[keep]
fc7 = fc7[keep,:]
assert(0 in keep)
return(rois,roi_scores, fc7)
else:
keep_all = keep_all2
new_nms_thresh += 0.1
return(rois,roi_scores, fc7)
def FasterRCNN_demo():
"""
Demo code
"""
DATA_DIR = 'data/'
demonet = 'res152_COCO'
tf.reset_default_graph(
) # Needed to use different nets one after the other
print(demonet)
if 'VOC' in demonet:
CLASSES = CLASSES_SET['VOC']
anchor_scales = [8, 16,
32] # It is needed for the right net architecture !!
elif 'COCO' in demonet:
CLASSES = CLASSES_SET['COCO']
anchor_scales = [4, 8, 16, 32]
nbClasses = len(CLASSES)
tfmodel = os.path.join(path_to_model, NETS_Pretrained[demonet])
#tfmodel = os.path.join(path_to_model,DATASETS[dataset][0],NETS[demonet][0])
print(tfmodel)
# tfmodel = os.path.join('output', demonet, DATASETS[dataset][0], 'default',
# NETS[demonet][0])
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True
# init session
sess = tf.Session(config=tfconfig)
# load network
if 'vgg16' in demonet:
net = vgg16()
elif demonet == 'res50':
raise NotImplementedError
elif 'res101' in demonet:
net = resnetv1(num_layers=101)
elif 'res152' in demonet:
net = resnetv1(num_layers=152)
elif demonet == 'mobile':
raise NotImplementedError
else:
raise NotImplementedError
net.create_architecture("TEST",
nbClasses,
tag='default',
anchor_scales=anchor_scales)
saver = tf.train.Saver()
saver.restore(sess, tfmodel)
print('Loaded network {:s}'.format(tfmodel))
im_name = 'loulou.jpg'
print('Demo for data/demo/{}'.format(im_name))
imfile = os.path.join(DATA_DIR, im_name)
im = cv2.imread(imfile)
scores, boxes = im_detect(
sess, net, im) # Arguments: im (ndarray): a color image in BGR order
# Only single-image batch implemented !
print('scores.shape', scores.shape)
#print(scores)
CONF_THRESH = 0.8
NMS_THRESH = 0.5 # non max suppression
for cls_ind, cls in enumerate(CLASSES[1:]):
cls_ind += 1 # because we skipped background
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets, NMS_THRESH)
dets = dets[keep, :]
inds = np.where(dets[:, -1] >= CONF_THRESH)[0]
if (len(inds) > 0):
print('CLASSES[cls_ind]', CLASSES[cls_ind])
vis_detections(im, cls, dets, thresh=CONF_THRESH)
plt.show()
sess.close()
def eval_MAP_SaliencyMethods(database='IconArt_v1',metamodel='FasterRCNN',demonet='res152_COCO',
k_per_bag=300,SaliencyMethod='SmoothGrad'):
"""
The goal of this function is to compute the mAP of the saliency method for
classification ResNet
@param : SaliencyMethod : IntegratedGrad ou SmoothGrad pour le moment
"""
matplotlib.use('Agg')
save_data = False
ReDo = True
plot = False
TEST_NMS = 0.01
thresh_classif = 0.1
# Parameter for the classification network
target_dataset = 'IconArt_v1'
style_layers = []
features = 'activation_48'
normalisation = False
final_clf= 'LinearSVC' # Don t matter
source_dataset= 'ImageNet'
transformOnFinalLayer='GlobalAveragePooling2D'
final_clf = 'MLP2'
epochs = 20
optimizer = 'SGD'
return_best_model = True
batch_size= 16
dropout=None
regulOnNewLayer=None
nesterov=False
SGDmomentum=0.9
decay=1e-4
cropCenter = False
# Load ResNet50 normalisation statistics
opt_option = [0.1,0.01]
pretrainingModif = True
kind_method = 'FT'
computeGlobalVariance = False
constrNet = 'ResNet50'
sizeIm = 224
Net = constrNet
# Load the box proosals
item_name,path_to_img,default_path_imdb,classes,ext,num_classes,str_val,df_label,path_data,Not_on_NicolasPC = get_database(database)
imdb,list_im_withanno = get_imdb_and_listImagesInTestSet(database)
num_images_detect = len(list_im_withanno)
dict_rois = getDictBoxesProposals(database=target_dataset,k_per_bag=k_per_bag,\
metamodel=metamodel,demonet=demonet)
for_data_output = os.path.join(path_data,'dataSaliencyMap',SaliencyMethod)
im_with_boxes_output = os.path.join(path_data,'SaliencyMapImagesBoxes',SaliencyMethod)
print('===',im_with_boxes_output)
pathlib.Path(for_data_output).mkdir(parents=True, exist_ok=True)
pathlib.Path(im_with_boxes_output).mkdir(parents=True, exist_ok=True)
# Load Classification model
print('loading :',constrNet,computeGlobalVariance,kind_method,pretrainingModif,opt_option)
model = learn_and_eval(target_dataset,source_dataset,final_clf,features,\
constrNet,kind_method,style_layers=style_layers,
normalisation=normalisation,transformOnFinalLayer=transformOnFinalLayer,
batch_size_RF=16,epochs_RF=20,momentum=0.9,ReDo=False,
returnStatistics=True,cropCenter=cropCenter,\
computeGlobalVariance=computeGlobalVariance,\
epochs=epochs,optimizer=optimizer,opt_option=opt_option,
return_best_model=return_best_model,\
batch_size=batch_size,gridSearch=False,verbose=True)
SaliencyMapClass_tab = []
stdev_spread = 0.1
nsamples = 50
x_steps = 50
for j in range(num_classes):
SaliencyMapClass=getSaliencyMapClass(model,c_i=j,method=SaliencyMethod,\
stdev_spread=stdev_spread,nsamples=nsamples,x_steps=x_steps)
SaliencyMapClass_tab +=[SaliencyMapClass]
# list_gt_boxes_classes = []
candidate_boxes = [[] for _ in range(imdb.num_images)]
all_boxes_order = [[[] for _ in range(num_images_detect)] for _ in range(imdb.num_classes)]
# number_gt_boxes = 0
itera = 20
norm = True
t0 = time.time()
# Un peu plus de 1440 images
for i in range(imdb.num_images):
# complet_name_tab = ('.'.join(complet_name.split('.')[0:-1])).split('/')
im_path = imdb.image_path_at(i)
name_im = im_path.split('/')[-1]
if i%itera==0:
t1 = time.time()
print(i,name_im,'duration for ',itera,'iterations = ',str(t1-t0),'s')
t0 = time.time()
im = cv2.imread(im_path)
hauteur, largeur ,_ = im.shape
blobs, im_scales = get_blobs(im)
if database=='PeopleArt':
name_im = '/'.join(im_path.split('/')[-2:])
if database=='PeopleArt':
name_im= '.'.join(name_im.split('.')[0:-1])
else:
name_im = name_im.split('.')[0]
proposals_boxes = dict_rois[name_im]
if cropCenter:
image_array= load_and_crop_img(path=im_path,Net=Net,target_size=sizeIm,
crop_size=sizeIm,interpolation='lanczos:center')
# For VGG or ResNet with classification head size == 224
else:
image_array = load_resize_and_process_img(im_path,Net=Net,max_dim=sizeIm)
#print(np.max(image_array),np.min(image_array),np.mean(image_array),np.median(image_array))
#input('wait')
dict_sensitivity = {}
dict_sensitivity_path = os.path.join(for_data_output,name_im+'_dict_SaliencyMap'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+'.pkl')
if not(os.path.exists(dict_sensitivity_path)) or ReDo:
predictions = model.predict(image_array)[0]
dict_sensitivity['predictions'] = predictions
inds = np.where(predictions > thresh_classif)[0]
for ind in inds:
prediction = predictions[ind]
if np.isnan(prediction):
print('Prediction of ',name_im,'is nan !!!')
input('wait')
candidate_boxes = []
j = ind +1 # the class index for the evaluation part
Smap=SaliencyMapClass_tab[ind].GetMask(image_array)
#print('before normalisation',np.max(Smap),np.min(Smap),np.mean(Smap),np.median(Smap))
if save_data: dict_sensitivity[j] = Smap
if SaliencyMethod=='SmoothGrad':
#Smap_grey = np.mean(Smap,axis=-1,keepdims=True)
Smap_grey = np.mean(np.abs(Smap),axis=-1,keepdims=True)
#print('after grey',np.max(Smap_grey),np.min(Smap_grey),np.mean(Smap_grey),np.median(Smap_grey))
if norm:
Smap_grey = to_01(Smap_grey)
#print('after normalisation',np.max(Smap_grey),np.min(Smap_grey),np.mean(Smap_grey),np.median(Smap_grey))
Smap_grey_time_score = prediction*Smap_grey
else: # In the case of Integrated Gradient
# Sur conseil d Antoine Pirovano
ptile= 99
# Sum for grayscale of the absolute value
pixel_attrs = np.sum(np.abs(Smap), axis=-1,keepdims=True)
pixel_attrs = np.clip(pixel_attrs / np.percentile(pixel_attrs, ptile), 0, 1)
Smap_grey_time_score = prediction * pixel_attrs
#print('after mul score',np.max(Smap_grey_time_score),np.min(Smap_grey_time_score),np.mean(Smap_grey_time_score),np.median(Smap_grey_time_score))
# attention truc super contre intuitif dans le resize c'est hauteur largeur alors que
# la fonction size retourne largeur hauteur
Smap_grey_time_score = Smap_grey_time_score[0]
#Smap_grey_time_score_resized = cv2.resize(Smap_grey_time_score, (hauteur, largeur),interpolation=cv2.INTER_NEAREST)
Smap_grey_time_score_resized = cv2.resize(Smap_grey_time_score, (largeur,hauteur),interpolation=cv2.INTER_NEAREST)
#print('Smap_grey_time_score_resized',Smap_grey_time_score_resized.shape,im.shape)
#print('after resize',np.max(Smap_grey_time_score_resized),np.min(Smap_grey_time_score_resized),np.mean(Smap_grey_time_score_resized),np.median(Smap_grey_time_score_resized))
if plot:
name_output = name_im+'_'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+ '_'+str(j)+'.jpg'
name_output_path = os.path.join(im_with_boxes_output,name_output)
Smap_grey_time_score_resized_01 = to_01(Smap_grey_time_score_resized)
plt.imshow(Smap_grey_time_score_resized_01, cmap=cm.gray)
plt.title(classes[j-1]+' : '+str(prediction))
plt.savefig(name_output_path)
plt.close()
for k in range(len(proposals_boxes)):
box = proposals_boxes[k]
x1,y1,x2,y2 = box # x : largeur et y en hauteur
x1_int = int(np.round(x1))
x2_int = int(np.round(x2))
y1_int = int(np.round(y1))
y2_int = int(np.round(y2))
#print(name_im,'Smap_grey_time_score_resized',Smap_grey_time_score_resized.shape,im.shape)
#print(x1_int,x2_int,y1_int,y2_int)
assert(x2_int<=largeur)
assert(y2_int<=hauteur)
Smap_grey_time_score_resized_crop = Smap_grey_time_score_resized[y1_int:y2_int,x1_int:x2_int]
# because bbox = dets[i, :4] # Boxes are score, x1,y1,x2,y2
Smap_grey_time_score_resized_crop_score = np.mean(Smap_grey_time_score_resized_crop)
# if k < 3:
# print('Smap_grey_time_score_resized_crop',Smap_grey_time_score_resized_crop.shape)
# print(x1_int,x2_int,y1_int,y2_int)
# print('j',j,'k',k,',score',Smap_grey_time_score_resized_crop_score)
if not(np.isnan(Smap_grey_time_score_resized_crop_score)):
box_with_scores = np.append(box,[Smap_grey_time_score_resized_crop_score])
candidate_boxes += [box_with_scores]
else:
box_with_scores = np.append(box,[0.0])
candidate_boxes += [box_with_scores]
# if np.isnan(Smap_grey_time_score_resized_crop_score):
# print('!!! score is nan')
# print(x1,y1,x2,y2)
# print(x1_int,x2_int,y1_int,y2_int)
# print(Smap_grey_time_score_resized_crop.shape)
# print(name_im,'Smap_grey_time_score_resized',Smap_grey_time_score_resized.shape,im.shape)
# print(prediction)
# print('after resize',np.max(Smap_grey_time_score_resized),np.min(Smap_grey_time_score_resized),np.mean(Smap_grey_time_score_resized),np.median(Smap_grey_time_score_resized))
# print(Smap_grey_time_score_resized_crop_score)
# input('wait')
#print(candidate_boxes)
if len(candidate_boxes)>0:
candidate_boxes_NP = np.array(candidate_boxes)
candidate_boxes_NP[:,-1] = candidate_boxes_NP[:,-1] -np.max(candidate_boxes_NP[:,-1]) + prediction
keep = nms(candidate_boxes_NP, TEST_NMS)
cls_dets = candidate_boxes_NP[keep, :]
all_boxes_order[j][i] = cls_dets
if plot:
roi_boxes_and_score = []
local_cls = []
for j in range(num_classes):
cls_dets = all_boxes_order[j+1][i]
if len(cls_dets) > 0:
local_cls += [classes[j]]
roi_boxes_score = cls_dets
if roi_boxes_and_score is None:
roi_boxes_and_score = [roi_boxes_score]
else:
roi_boxes_and_score += [roi_boxes_score]
if roi_boxes_and_score is None: roi_boxes_and_score = [[]]
#print(name_im,roi_boxes_and_score,local_cls)
vis_detections_list(im, local_cls, roi_boxes_and_score, thresh=-np.inf)
name_output = name_im+'_'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+ '_Regions.jpg'
name_output_path = os.path.join(im_with_boxes_output,name_output)
#input("wait")
plt.savefig(name_output_path)
plt.close()
if save_data:
with open(dict_sensitivity_path, 'wb') as f:
pickle.dump(dict_sensitivity, f, pickle.HIGHEST_PROTOCOL)
# for i in range(imdb.num_images):
# candidate_boxes[i] = np.array(candidate_boxes[i])
imdb.set_force_dont_use_07_metric(True)
det_file = os.path.join(path_data, 'detectionsSaliencyMap'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+'.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data +'tmp/' + database+'_'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+'_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir) # AP at O.5
print("===> Detection score (thres = 0.5): ",database,'with Saliency map from',SaliencyMethod,'with std =',stdev_spread,'nsamples = ',nsamples,'x_steps =',x_steps)
print(arrayToLatex(aps,per=True))
ovthresh_tab = [0.3,0.1,0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order, output_dir,ovthresh)
print("Detection score with thres at ",ovthresh)
print(arrayToLatex(aps,per=True))
def tfR_evaluation_parall(database,
num_classes,
export_dir,
dict_name_file,
mini_batch_size,
config,
scoreInMI_max,
path_to_img,
path_data,
classes,
verbose,
thresh_evaluation,
TEST_NMS,
all_boxes=None,
PlotRegions=False,
cachefile_model_base='',
number_im=np.inf,
dim_rois=5,
usecache=True,
k_per_bag=300,
num_features=2048):
"""
@param : number_im : number of image plot at maximum
"""
if verbose:
print('thresh_evaluation', thresh_evaluation, 'TEST_NMS', TEST_NMS)
thresh = thresh_evaluation # Threshold score or distance MI_max
#TEST_NMS = 0.7 # Recouvrement entre les classes
load_model = False
with_tanh = True
if PlotRegions:
extensionStocha = os.path.join(cachefile_model_base, 'ForIllustraion')
path_to_output2 = os.path.join(path_data, 'tfMI_maxRegion', database,
extensionStocha)
path_to_output2_bis = os.path.join(path_to_output2, 'Train')
path_to_output2_ter = os.path.join(path_to_output2, 'Test')
pathlib.Path(path_to_output2_bis).mkdir(parents=True, exist_ok=True)
pathlib.Path(path_to_output2_ter).mkdir(parents=True, exist_ok=True)
listexportsplit = export_dir.split('/')[:-1]
export_dir_path = os.path.join(*listexportsplit)
name_model_meta = export_dir + '.meta'
get_roisScore = scoreInMI_max
if PlotRegions:
index_im = 0
if verbose:
print(
"Start ploting Regions selected by the MI_max in training phase"
)
train_dataset = tf.data.TFRecordDataset(dict_name_file['trainval'])
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r, \
num_classes=num_classes,with_rois_scores=get_roisScore,num_features=num_features,\
num_rois=k_per_bag,dim_rois=dim_rois))
dataset_batch = train_dataset.batch(mini_batch_size)
if usecache:
dataset_batch.cache()
iterator = dataset_batch.make_one_shot_iterator()
next_element = iterator.get_next()
with tf.Session(config=config) as sess:
new_saver = tf.train.import_meta_graph(name_model_meta)
new_saver.restore(sess,
tf.train.latest_checkpoint(export_dir_path))
load_model = True
graph = tf.get_default_graph()
X = get_tensor_by_nameDescendant(graph, "X")
y = get_tensor_by_nameDescendant(graph, "y")
if scoreInMI_max:
scores_tf = get_tensor_by_nameDescendant(graph, "scores")
Prod_best = get_tensor_by_nameDescendant(graph, "ProdScore")
else:
Prod_best = get_tensor_by_nameDescendant(graph, "Prod")
if with_tanh:
if verbose: print('use of tanh')
Tanh = tf.tanh(Prod_best)
mei = tf.argmax(Tanh, axis=2)
score_mei = tf.reduce_max(Tanh, axis=2)
else:
mei = tf.argmax(Prod_best, axis=2)
score_mei = tf.reduce_max(Prod_best, axis=2)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
while True:
try:
next_element_value = sess.run(next_element)
if not (scoreInMI_max):
fc7s, roiss, labels, name_imgs = next_element_value
else:
fc7s, roiss, rois_scores, labels, name_imgs = next_element_value
if scoreInMI_max:
feed_dict_value = {
X: fc7s,
scores_tf: rois_scores,
y: labels
}
else:
feed_dict_value = {X: fc7s, y: labels}
if with_tanh:
PositiveRegions,get_PositiveRegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Tanh], feed_dict=feed_dict_value)
else:
PositiveRegions,get_PositiveRegionsScore,PositiveExScoreAll = \
sess.run([mei,score_mei,Prod_best], feed_dict=feed_dict_value)
print('Start plotting Training exemples')
for k in range(len(labels)):
if index_im > number_im:
continue
if database in [
'IconArt_v1', 'clipart', 'watercolor',
'PeopleArt'
]:
name_img = str(name_imgs[k].decode("utf-8"))
else:
name_img = name_imgs[k]
rois = roiss[k, :]
#if verbose: print(name_img)
if database in [
'IconArt_v1', 'clipart', 'watercolor',
'PeopleArt'
]:
complet_name = os.path.join(
path_to_img, name_img + '.jpg')
name_sans_ext = name_img
else:
name_sans_ext = os.path.splitext(name_img)[0]
complet_name = os.path.join(
path_to_img, name_sans_ext + '.jpg')
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
scores_all = PositiveExScoreAll[:, k, :]
roi = roiss[k, :]
if dim_rois == 5:
roi_boxes = roi[:, 1:5] / im_scales[0]
else:
roi_boxes = roi / im_scales[0]
roi_boxes_and_score = None
local_cls = []
for j in range(num_classes):
if labels[k, j] == 1:
local_cls += [classes[j]]
roi_with_object_of_the_class = PositiveRegions[
j, k] % len(
rois
) # Because we have repeated some rois
roi = rois[roi_with_object_of_the_class, :]
roi_scores = [get_PositiveRegionsScore[j, k]]
if dim_rois == 5:
roi_boxes = roi[1:5] / im_scales[0]
else:
roi_boxes = roi / im_scales[0]
roi_boxes_score = np.expand_dims(
np.expand_dims(np.concatenate(
(roi_boxes, roi_scores)),
axis=0),
axis=0)
if roi_boxes_and_score is None:
roi_boxes_and_score = roi_boxes_score
else:
roi_boxes_and_score= \
np.vstack((roi_boxes_and_score,roi_boxes_score))
cls = local_cls
if roi_boxes_and_score is None:
roi_boxes_and_score = []
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=-np.inf)
name_output = os.path.join(
path_to_output2, 'Train',
name_sans_ext + '_Regions.jpg')
if database == 'PeopleArt':
list_split = name_output.split('/')[0:-1]
path_tmp = os.path.join(*list_split)
pathlib.Path(path_tmp).mkdir(parents=True,
exist_ok=True)
plt.savefig(name_output)
plt.close()
index_im += 1
except tf.errors.OutOfRangeError:
break
if verbose: print("Testing Time")
# Testing time !
train_dataset = tf.data.TFRecordDataset(dict_name_file['test'])
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore,num_features=num_features,
num_rois=k_per_bag,dim_rois=dim_rois))
dataset_batch = train_dataset.batch(mini_batch_size)
if usecache:
dataset_batch.cache()
iterator = dataset_batch.make_one_shot_iterator()
next_element = iterator.get_next()
true_label_all_test = []
predict_label_all_test = []
name_all_test = []
FirstTime = True
i = 0
ii = 0
with tf.Session(config=config) as sess:
if load_model == False:
new_saver = tf.train.import_meta_graph(name_model_meta)
new_saver.restore(sess,
tf.train.latest_checkpoint(export_dir_path))
graph = tf.get_default_graph()
X = get_tensor_by_nameDescendant(graph, "X")
y = get_tensor_by_nameDescendant(graph, "y")
if scoreInMI_max:
scores_tf = get_tensor_by_nameDescendant(graph, "scores")
Prod_best = get_tensor_by_nameDescendant(graph, "ProdScore")
else:
Prod_best = get_tensor_by_nameDescendant(graph, "Prod")
if with_tanh:
print(
'We add the tanh in the test fct to get score between -1 and 1.'
)
Tanh = tf.tanh(Prod_best)
mei = tf.argmax(Tanh, axis=2)
score_mei = tf.reduce_max(Tanh, axis=2)
else:
mei = tf.argmax(Prod_best, axis=-1)
score_mei = tf.reduce_max(Prod_best, axis=-1)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Evaluation Test : Probleme ici souvent
while True:
try:
if not (scoreInMI_max):
fc7s, roiss, labels, name_imgs = sess.run(next_element)
else:
fc7s, roiss, rois_scores, labels, name_imgs = sess.run(
next_element)
if scoreInMI_max:
feed_dict_value = {
X: fc7s,
scores_tf: rois_scores,
y: labels
}
else:
feed_dict_value = {X: fc7s, y: labels}
if with_tanh:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Tanh], feed_dict=feed_dict_value)
true_label_all_test += [labels]
predict_label_all_test += [get_RegionsScore
] # For the classification task
for k in range(len(labels)):
if database in [
'IconArt_v1', 'watercolor', 'clipart', 'PeopleArt'
]:
complet_name = os.path.join(
path_to_img,
str(name_imgs[k].decode("utf-8")) + '.jpg')
else:
complet_name = os.path.join(path_to_img,
name_imgs[k] + '.jpg')
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
scores_all = PositiveExScoreAll[:, k, :]
roi = roiss[k, :]
if dim_rois == 5:
roi_boxes = roi[:, 1:5] / im_scales[0]
else:
roi_boxes = roi / im_scales[0]
for j in range(num_classes):
scores = scores_all[j, :]
inds = np.where(scores > thresh)[0]
cls_scores = scores[inds]
cls_boxes = roi_boxes[inds, :]
cls_dets = np.hstack(
(cls_boxes,
cls_scores[:, np.newaxis])).astype(np.float32,
copy=False)
keep = nms(cls_dets, TEST_NMS)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
i += 1
for l in range(len(name_imgs)):
if database in [
'IconArt_v1', 'watercolor', 'clipart', 'PeopleArt'
]:
name_all_test += [[str(name_imgs[l].decode("utf-8"))]]
else:
name_all_test += [[name_imgs[l]]]
if PlotRegions:
if verbose and (ii % 1000 == 0):
print("Plot the images :", ii)
if verbose and FirstTime:
FirstTime = False
print("Start ploting Regions on test set")
for k in range(len(labels)):
if ii > number_im:
continue
if database in [
'IconArt_v1', 'watercolor', 'clipart',
'PeopleArt'
]:
name_img = str(name_imgs[k].decode("utf-8"))
else:
name_img = name_imgs[k]
rois = roiss[k, :]
if database in [
'IconArt_v1', 'watercolor', 'clipart',
'PeopleArt'
]:
complet_name = os.path.join(
path_to_img, name_img + '.jpg')
name_sans_ext = name_img
elif (database == 'Wikidata_Paintings') or (
database
== 'Wikidata_Paintings_miniset_verif'):
name_sans_ext = os.path.splitext(name_img)[0]
complet_name = os.path.join(
path_to_img, name_sans_ext + '.jpg')
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
roi_boxes_and_score = []
local_cls = []
for j in range(num_classes):
cls_dets = all_boxes[j][
ii] # Here we have #classe x box dim + score
if len(cls_dets) > 0:
local_cls += [classes[j]]
roi_boxes_score = cls_dets
if roi_boxes_and_score is None:
roi_boxes_and_score = [roi_boxes_score]
else:
roi_boxes_and_score += [roi_boxes_score]
if roi_boxes_and_score is None:
roi_boxes_and_score = [[]]
ii += 1
cls = local_cls
# In this case we will plot several version of the image
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=-np.inf)
name_output = os.path.join(
path_to_output2, 'Test',
name_sans_ext + '_Regions.jpg')
if database == 'PeopleArt':
list_split = name_output.split('/')[0:-1]
path_tmp = os.path.join(*list_split)
pathlib.Path(path_tmp).mkdir(parents=True,
exist_ok=True)
plt.savefig(name_output)
plt.close()
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=0.25)
name_output = os.path.join(
path_to_output2, 'Test',
name_sans_ext + '_Regions_over025.jpg')
plt.savefig(name_output)
plt.close()
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=0.5)
name_output = os.path.join(
path_to_output2, 'Test',
name_sans_ext + '_Regions_over05.jpg')
plt.savefig(name_output)
plt.close()
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=0.75)
name_output = os.path.join(
path_to_output2, 'Test',
name_sans_ext + '_Regions_over075.jpg')
plt.savefig(name_output)
plt.close()
except tf.errors.OutOfRangeError:
break
tf.reset_default_graph()
true_label_all_test = np.concatenate(true_label_all_test)
predict_label_all_test = np.transpose(
np.concatenate(predict_label_all_test, axis=1))
name_all_test = np.concatenate(name_all_test)
labels_test_predited = (np.sign(predict_label_all_test) + 1.) / 2
labels_test_predited[np.where(
labels_test_predited ==
0.5)] = 0 # To deal with the case where predict_label_all_test == 0
return (true_label_all_test, predict_label_all_test, name_all_test,
labels_test_predited, all_boxes)
def plot_Train_Test_Regions(database,
number_im,
dict_name_file,
path_to_output2,
export_dir,
problem_class=[],
RPN=False,
water_mark='',
transform_output=None,
with_rois_scores_atEnd=False,
scoreInMILSVM=False):
verbose = True
# k_per_bag,positive_elt,size_output = param_clf
thresh = 0.0 # Threshold score or distance MILSVM
TEST_NMS = 0.3 # Recouvrement entre les classes
mini_batch_size = 100
load_model = False
if database == 'VOC2007':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/VOCdevkit/VOC2007/JPEGImages/'
classes = [
'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car',
'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',
'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'
]
elif database == 'watercolor':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/cross-domain-detection/datasets/watercolor/JPEGImages/'
classes = ['bicycle', 'bird', 'car', 'cat', 'dog', 'person']
else:
raise (NotImplemented)
if len(problem_class) > 0:
label_problem = []
for c in problem_class:
label_problem += [np.where(np.array(classes) == c)[0][0]]
else:
label_problem = np.arange(len(classes))
num_classes = len(classes)
num_images = 10000
all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]
path_to_output2_bis = path_to_output2 + 'Train/'
path_to_output2_ter = path_to_output2 + 'Test/'
print(path_to_output2_bis, path_to_output2_ter)
pathlib.Path(path_to_output2_bis).mkdir(parents=True, exist_ok=True)
pathlib.Path(path_to_output2_ter).mkdir(parents=True, exist_ok=True)
export_dir_path = ('/').join(export_dir.split('/')[:-1])
name_model_meta = export_dir + '.meta'
get_roisScore = (with_rois_scores_atEnd or scoreInMILSVM)
if verbose:
print("Start ploting Regions selected by the MILSVM in training phase")
if transform_output == 'tanh':
with_tanh = True
with_softmax = False
elif transform_output == 'softmax':
with_softmax = True
with_tanh = False
# if seuil_estimation: print('It may cause problem of doing softmax and tangent estimation')
else:
with_softmax, with_tanh = False, False
train_dataset = tf.data.TFRecordDataset(dict_name_file['trainval'])
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore))
dataset_batch = train_dataset.batch(mini_batch_size)
dataset_batch.cache()
iterator = dataset_batch.make_one_shot_iterator()
next_element = iterator.get_next()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
new_saver = tf.train.import_meta_graph(name_model_meta)
new_saver.restore(sess, tf.train.latest_checkpoint(export_dir_path))
load_model = True
graph = tf.get_default_graph()
X = graph.get_tensor_by_name("X:0")
y = graph.get_tensor_by_name("y:0")
if scoreInMILSVM:
scores_tf = graph.get_tensor_by_name("scores:0")
Prod_best = graph.get_tensor_by_name("ProdScore:0")
else:
Prod_best = graph.get_tensor_by_name("Prod:0")
if with_tanh:
print('use of tanh')
Tanh = tf.tanh(Prod_best)
mei = tf.argmax(Tanh, axis=2)
score_mei = tf.reduce_max(Tanh, axis=2)
elif with_softmax:
Softmax = tf.nn.softmax(Prod_best, axis=-1)
mei = tf.argmax(Softmax, axis=2)
score_mei = tf.reduce_max(Softmax, axis=2)
else:
mei = tf.argmax(Prod_best, axis=2)
score_mei = tf.reduce_max(Prod_best, axis=2)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
index_im = 0
while True:
try:
next_element_value = sess.run(next_element)
# print(len(next_element_value))
if not (with_rois_scores_atEnd) and not (scoreInMILSVM):
fc7s, roiss, labels, name_imgs = next_element_value
else:
fc7s, roiss, rois_scores, labels, name_imgs = next_element_value
if scoreInMILSVM:
feed_dict_value = {
X: fc7s,
scores_tf: rois_scores,
y: labels
}
else:
feed_dict_value = {X: fc7s, y: labels}
if with_tanh:
PositiveRegions,get_PositiveRegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Tanh], feed_dict=feed_dict_value)
elif with_softmax:
PositiveRegions,get_PositiveRegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Softmax], feed_dict=feed_dict_value)
else:
PositiveRegions,get_PositiveRegionsScore,PositiveExScoreAll = \
sess.run([mei,score_mei,Prod_best], feed_dict=feed_dict_value)
for k in range(len(labels)):
label_tmp = np.array(labels[k, :])
label_tmp_index = np.where(label_tmp == 1)
intersec = np.intersect1d(label_tmp_index, label_problem)
# print(intersec)
if len(intersec) == 0:
continue
if index_im > number_im:
continue
if database == 'VOC2007' or database == 'watercolor':
name_img = str(name_imgs[k].decode("utf-8"))
else:
name_img = name_imgs[k]
rois = roiss[k, :]
#if verbose: print(name_img)
if database == 'VOC12' or database == 'Paintings' or database == 'VOC2007' or database == 'watercolor':
complet_name = path_to_img + name_img + '.jpg'
name_sans_ext = name_img
elif (database == 'Wikidata_Paintings') or (
database == 'Wikidata_Paintings_miniset_verif'):
name_sans_ext = os.path.splitext(name_img)[0]
complet_name = path_to_img + name_sans_ext + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
scores_all = PositiveExScoreAll[:, k, :]
roi = roiss[k, :]
roi_boxes = roi[:, 1:5] / im_scales[0]
roi_boxes_and_score = None
local_cls = []
for j in range(num_classes):
if labels[k, j] == 1:
local_cls += [classes[j]]
roi_with_object_of_the_class = PositiveRegions[
j, k] % len(
rois) # Because we have repeated some rois
roi = rois[roi_with_object_of_the_class, :]
roi_scores = [get_PositiveRegionsScore[j, k]]
roi_boxes = roi[1:5] / im_scales[0]
roi_boxes_score = np.expand_dims(np.expand_dims(
np.concatenate((roi_boxes, roi_scores)),
axis=0),
axis=0)
if roi_boxes_and_score is None:
roi_boxes_and_score = roi_boxes_score
else:
roi_boxes_and_score= \
np.vstack((roi_boxes_and_score,roi_boxes_score))
if RPN:
best_RPN_roi = rois[0, :]
best_RPN_roi_boxes = best_RPN_roi[1:5] / im_scales[0]
best_RPN_roi_scores = [PositiveExScoreAll[j, k, 0]]
cls = local_cls + [
'RPN'
] # Comparison of the best region according to the faster RCNN and according to the MILSVM de Said
best_RPN_roi_boxes_score = np.expand_dims(
np.expand_dims(np.concatenate(
(best_RPN_roi_boxes, best_RPN_roi_scores)),
axis=0),
axis=0)
roi_boxes_and_score = np.vstack(
(roi_boxes_and_score, best_RPN_roi_boxes_score))
else:
cls = local_cls
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=-np.inf)
name_output = path_to_output2_bis + name_sans_ext + water_mark + '_Regions.jpg'
plt.savefig(name_output)
plt.close()
index_im += 1
except tf.errors.OutOfRangeError:
break
#tf.reset_default_graph()
print("Testing Time")
# Training time !
# Testing time !
train_dataset = tf.data.TFRecordDataset(dict_name_file['test'])
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore))
dataset_batch = train_dataset.batch(mini_batch_size)
dataset_batch.cache()
iterator = dataset_batch.make_one_shot_iterator()
next_element = iterator.get_next()
FirstTime = True
i = 0
ii = 0
with tf.Session(config=config) as sess:
if load_model == False:
new_saver = tf.train.import_meta_graph(name_model_meta)
new_saver.restore(sess,
tf.train.latest_checkpoint(export_dir_path))
graph = tf.get_default_graph()
X = graph.get_tensor_by_name("X:0")
y = graph.get_tensor_by_name("y:0")
if scoreInMILSVM:
scores_tf = graph.get_tensor_by_name("scores:0")
Prod_best = graph.get_tensor_by_name("ProdScore:0")
else:
Prod_best = graph.get_tensor_by_name("Prod:0")
if with_tanh:
print('use of tanh')
Tanh = tf.tanh(Prod_best)
mei = tf.argmax(Tanh, axis=2)
score_mei = tf.reduce_max(Tanh, axis=2)
elif with_softmax:
Softmax = tf.nn.softmax(Prod_best, axis=-1)
mei = tf.argmax(Softmax, axis=2)
score_mei = tf.reduce_max(Softmax, axis=2)
else:
mei = tf.argmax(Prod_best, axis=2)
score_mei = tf.reduce_max(Prod_best, axis=2)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
while True:
try:
if not (with_rois_scores_atEnd) and not (scoreInMILSVM):
fc7s, roiss, labels, name_imgs = sess.run(next_element)
else:
fc7s, roiss, rois_scores, labels, name_imgs = sess.run(
next_element)
if scoreInMILSVM:
feed_dict_value = {
X: fc7s,
scores_tf: rois_scores,
y: labels
}
else:
feed_dict_value = {X: fc7s, y: labels}
if with_tanh:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Tanh], feed_dict=feed_dict_value)
elif with_softmax:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Softmax], feed_dict=feed_dict_value)
else:
PositiveRegions,get_RegionsScore,PositiveExScoreAll = \
sess.run([mei,score_mei,Prod_best], feed_dict=feed_dict_value)
if with_rois_scores_atEnd:
PositiveExScoreAll = PositiveExScoreAll * rois_scores
get_RegionsScore = np.max(PositiveExScoreAll, axis=2)
PositiveRegions = np.amax(PositiveExScoreAll, axis=2)
# if predict_with=='LinearSVC':
for k in range(len(labels)):
label_tmp = np.array(labels[k, :])
label_tmp_index = np.where(label_tmp == 1)
intersec = np.intersect1d(label_tmp_index, label_problem)
if len(intersec) == 0:
continue
if database == 'VOC2007' or database == 'watercolor':
complet_name = path_to_img + str(
name_imgs[k].decode("utf-8")) + '.jpg'
else:
complet_name = path_to_img + name_imgs[k] + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
scores_all = PositiveExScoreAll[:, k, :]
roi = roiss[k, :]
roi_boxes = roi[:, 1:5] / im_scales[0]
for j in range(num_classes):
scores = scores_all[j, :]
inds = np.where(scores > thresh)[0]
cls_scores = scores[inds]
cls_boxes = roi_boxes[inds, :]
cls_dets = np.hstack(
(cls_boxes,
cls_scores[:, np.newaxis])).astype(np.float32,
copy=False)
keep = nms(cls_dets, TEST_NMS)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
i += 1
# for l in range(len(name_imgs)):
# if database=='VOC2007' :
# name_all_test += [[str(name_imgs[l].decode("utf-8"))]]
# else:
# name_all_test += [[name_imgs[l]]]
if verbose and (ii % 1000 == 0):
print("Plot the images :", ii)
if verbose and FirstTime:
FirstTime = False
print("Start ploting Regions on test set")
for k in range(len(labels)):
if ii > number_im:
continue
if database == 'VOC2007' or database == 'watercolor':
name_img = str(name_imgs[k].decode("utf-8"))
else:
name_img = name_imgs[k]
rois = roiss[k, :]
if database == 'VOC12' or database == 'Paintings' or database == 'VOC2007' or database == 'watercolor':
complet_name = path_to_img + name_img + '.jpg'
name_sans_ext = name_img
elif (database == 'Wikidata_Paintings') or (
database == 'Wikidata_Paintings_miniset_verif'):
name_sans_ext = os.path.splitext(name_img)[0]
complet_name = path_to_img + name_sans_ext + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
roi_boxes_and_score = []
local_cls = []
for j in range(num_classes):
cls_dets = all_boxes[j][
ii] # Here we have #classe x box dim + score
# Last element is the score
# print(cls_dets.shape)
if len(cls_dets) > 0:
local_cls += [classes[j]]
# roi_boxes_score = np.expand_dims(cls_dets,axis=0)
roi_boxes_score = cls_dets
# print(roi_boxes_score.shape)
if roi_boxes_and_score is None:
roi_boxes_and_score = [roi_boxes_score]
else:
roi_boxes_and_score += [roi_boxes_score]
#np.vstack((roi_boxes_and_score,roi_boxes_score))
if roi_boxes_and_score is None: roi_boxes_and_score = [[]]
ii += 1
if RPN:
best_RPN_roi = rois[0, :]
best_RPN_roi_boxes = best_RPN_roi[1:5] / im_scales[0]
best_RPN_roi_scores = [PositiveExScoreAll[j, k, 0]]
cls = local_cls + [
'RPN'
] # Comparison of the best region according to the faster RCNN and according to the MILSVM de Said
#best_RPN_roi_boxes_score = np.expand_dims(np.expand_dims(np.concatenate((best_RPN_roi_boxes,best_RPN_roi_scores)),axis=0),axis=0)
best_RPN_roi_boxes_score = np.expand_dims(
np.concatenate(
(best_RPN_roi_boxes, best_RPN_roi_scores)),
axis=0)
# print(best_RPN_roi_boxes_score.shape)
#roi_boxes_and_score = np.vstack((roi_boxes_and_score,best_RPN_roi_boxes_score))
roi_boxes_and_score += [
best_RPN_roi_boxes_score
] #np.vstack((roi_boxes_and_score,best_RPN_roi_boxes_score))
else:
cls = local_cls
#print(len(cls),len(roi_boxes_and_score))
# Attention you use the 0.5 threshold
vis_detections_list(im,
cls,
roi_boxes_and_score,
thresh=0.0)
name_output = path_to_output2_ter + name_sans_ext + water_mark + '_Regions.jpg'
plt.savefig(name_output)
plt.close()
except tf.errors.OutOfRangeError:
break
tf.reset_default_graph()
def mainEval(dataset_nm='IconArt_v1',classe=0,k_per_bag = 300,metamodel = 'FasterRCNN',\
demonet='res152_COCO',test=False,MILmodel='MI_Net',max_epoch=20,verbose=True):
# dataset_nm='IconArt_v1'
# classe=1
# k_per_bag = 300
# metamodel = 'FasterRCNN'
# demonet='res152_COCO'
# test=True
# MILmodel='MI_Net_with_DS'
# max_epoch = 1
t0 = time.time()
if test:
classe = 0
if MILmodel=='MI_Net':
MILmodel_fct = MI_Net_WSOD
elif MILmodel=='MI_Max_AddOneLayer_Keras':
MILmodel_fct = MI_Max_AddOneLayer_Keras
elif MILmodel=='mi_Net':
MILmodel_fct = mi_Net_WSOD
elif MILmodel=='MI_Net_with_DS':
MILmodel_fct = MI_Net_with_DS_WSOD
elif MILmodel=='MI_Net_with_RC':
MILmodel_fct = MI_Net_with_RC_WSOD
else:
print(MILmodel,'is unkwon')
return(0)
print('MILmodel',MILmodel,max_epoch)
item_name,path_to_img,default_path_imdb,classes,ext,num_classes,str_val,df_label,\
path_data,Not_on_NicolasPC = get_database(dataset_nm)
dataset,bags_full_label,mean_fea,std_fea = load_dataset(dataset_nm,classe=0,k_per_bag = k_per_bag,metamodel = metamodel,demonet=demonet)
model_dict = {}
for j in range(num_classes):
if test and not(j==classe):
continue
else:
for k in range(len(dataset['train'])):
a = list(dataset['train'][k])
a[1] = [bags_full_label[k,j]]*k_per_bag
a = tuple(a)
dataset['train'][k] = a
print('start training for class',j)
model = MILmodel_fct(dataset,max_epoch=max_epoch,verbose=verbose)
model_dict[j] = model
t1 = time.time()
print("--- Training duration :",str(t1-t0),' s')
dict_name_file = getDictFeaturesPrecomputed(dataset_nm,k_per_bag=k_per_bag,\
metamodel=metamodel,demonet=demonet)
name_file = dict_name_file['test']
if metamodel=='EdgeBoxes':
dim_rois = 4
else:
dim_rois = 5
next_element = getTFRecordDataset(name_file,k_per_bag =k_per_bag,dim_rois = dim_rois,
num_classes = num_classes)
dont_use_07_metric = False
if dataset_nm=='VOC2007':
imdb = get_imdb('voc_2007_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='watercolor':
imdb = get_imdb('watercolor_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='PeopleArt':
imdb = get_imdb('PeopleArt_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='clipart':
imdb = get_imdb('clipart_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='comic':
imdb = get_imdb('comic_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='CASPApaintings':
imdb = get_imdb('CASPApaintings_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='IconArt_v1' or dataset_nm=='RMN':
imdb = get_imdb('IconArt_v1_test',data_path=default_path_imdb)
num_images = len(df_label[df_label['set']=='test'][item_name])
elif 'IconArt_v1' in dataset_nm and not('IconArt_v1' ==dataset_nm):
imdb = get_imdb('IconArt_v1_test',ext=dataset_nm.split('_')[-1],data_path=default_path_imdb)
# num_images = len(imdb.image_index)
num_images = len(df_label[df_label['set']=='test'][item_name])
elif dataset_nm in ['WikiTenLabels','MiniTrain_WikiTenLabels','WikiLabels1000training']:
imdb = get_imdb('WikiTenLabels_test',data_path=default_path_imdb)
#num_images = len(imdb.image_index)
num_images = len(df_label[df_label['set']=='test'][item_name])
elif 'OIV5' in dataset_nm: # For OIV5 for instance !
num_images = len(df_label[df_label['set']=='test'][item_name])
else:
num_images = len(df_label[df_label['set']=='test'][item_name])
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]
TEST_NMS = 0.3
thresh = 0.0
true_label_all_test = []
predict_label_all_test = []
name_all_test = []
config = tf.ConfigProto()
config.intra_op_parallelism_threads = 16
config.inter_op_parallelism_threads = 16
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
i = 0
num_features = 2048
while True:
try:
fc7s,roiss,rois_scores,labels,name_imgs = sess.run(next_element)
fc7s = np.divide(fc7s-mean_fea, std_fea).astype(np.float32)
true_label_all_test += [labels]
score_all = None
for j in range(num_classes):
if not(test):
model= model_dict[j]
predictions = model.predict(fc7s.reshape((-1,num_features)),batch_size=1)
if MILmodel=='MI_Net_with_DS':
predictions = predictions[3]
scores_all_j_k = predictions.reshape((fc7s.shape[0],1,fc7s.shape[1]))
else:
if j==classe:
model= model_dict[j]
predictions = model.predict(fc7s.reshape((-1,num_features)),batch_size=1)
if MILmodel=='MI_Net_with_DS':
predictions = predictions[3]
scores_all_j_k = predictions.reshape((fc7s.shape[0],1,fc7s.shape[1]))
if score_all is None:
score_all = scores_all_j_k
else:
score_all = np.concatenate((score_all,scores_all_j_k),axis=1)
predict_label_all_test += [np.max(score_all,axis=2)]
for k in range(len(labels)):
name_im = name_imgs[k].decode("utf-8")
complet_name = path_to_img + str(name_im) + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
roi = roiss[k,:]
if metamodel=='EdgeBoxes':
roi_boxes = roi / im_scales[0]
else:
roi_boxes = roi[:,1:5] / im_scales[0]
for j in range(num_classes):
scores = score_all[k,j,:]
#print(j,'scores',scores.shape)
inds = np.where(scores > thresh)[0]
cls_scores = scores[inds]
cls_boxes = roi_boxes[inds,:]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32, copy=False)
keep = nms(cls_dets, TEST_NMS)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
i += 1
for l in range(len(name_imgs)):
if dataset_nm in ['IconArt_v1','VOC2007','watercolor','clipart',\
'comic','CASPApaintings','WikiTenLabels','PeopleArt',\
'MiniTrain_WikiTenLabels','WikiLabels1000training']:
name_all_test += [[str(name_imgs[l].decode("utf-8"))]]
else:
name_all_test += [[name_imgs[l]]]
except tf.errors.OutOfRangeError:
break
sess.close()
true_label_all_test = np.concatenate(true_label_all_test)
predict_label_all_test = np.concatenate(predict_label_all_test,axis=0)
name_all_test = np.concatenate(name_all_test)
AP_per_class = []
for j,classe in enumerate(classes):
AP = average_precision_score(true_label_all_test[:,j],predict_label_all_test[:,j],average=None)
AP_per_class += [AP]
print('Average Precision classification task :')
print(arrayToLatex(AP_per_class,per=True))
max_per_image = 100
num_images_detect = len(imdb.image_index) # We do not have the same number of images in the WikiTenLabels or IconArt_v1 case
all_boxes_order = [[[] for _ in range(num_images_detect)] for _ in range(imdb.num_classes)]
number_im = 0
name_all_test = name_all_test.astype(str)
for i in range(num_images_detect):
# print(i)
name_img = imdb.image_path_at(i)
if dataset_nm=='PeopleArt':
name_img_wt_ext = name_img.split('/')[-2] +'/' +name_img.split('/')[-1]
name_img_wt_ext_tab =name_img_wt_ext.split('.')
name_img_wt_ext = '.'.join(name_img_wt_ext_tab[0:-1])
else:
name_img_wt_ext = name_img.split('/')[-1]
name_img_wt_ext =name_img_wt_ext.split('.')[0]
name_img_ind = np.where(np.array(name_all_test)==name_img_wt_ext)[0]
#print(name_img_ind)
if len(name_img_ind)==0:
print('len(name_img_ind), images not found in the all_boxes')
print(name_img_wt_ext)
raise(Exception)
else:
number_im += 1
# print(name_img_ind[0])
for j in range(1, imdb.num_classes):
j_minus_1 = j-1
if len(all_boxes[j_minus_1][name_img_ind[0]]) >0:
all_boxes_order[j][i] = all_boxes[j_minus_1][name_img_ind[0]]
if max_per_image > 0 and len(all_boxes_order[j][i]) >0:
image_scores = np.hstack([all_boxes_order[j][i][:, -1]
for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes_order[j][i][:, -1] >= image_thresh)[0]
all_boxes_order[j][i] = all_boxes_order[j][i][keep, :]
assert (number_im==num_images_detect) # To check that we have the all the images in the detection prediction
det_file = os.path.join(path_data, 'detections.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data +'tmp/' + dataset_nm+'_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
apsAt05 = aps
print("Detection score (thres = 0.5): ",dataset_nm)
print(arrayToLatex(aps,per=True))
ovthresh_tab = [0.3,0.1,0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order, output_dir,ovthresh)
if ovthresh == 0.1:
apsAt01 = aps
print("Detection score with thres at ",ovthresh,'with ',MILmodel)
print(arrayToLatex(aps,per=True))
t2 = time.time()
print("--- Testing duration :",str(t2-t1),' s')
return(apsAt05,apsAt01,AP_per_class)
def LearnOn30andTestOn300():
""" This function is just used to do the test with a model learn on 30 regions"""
demonet = 'res152_COCO'
database = 'WikiTenLabels'
verbose = True
testMode = False
jtest = 'cow'
PlotRegions = False
saved_clf=False
RPN=False
CompBest=False
Stocha=True
k_per_bag=30
parallel_op=True
CV_Mode=''
num_split=2
WR=True
init_by_mean =None
seuil_estimation=''
restarts=11
max_iters_all_base=300
LR=0.01
with_tanh=True
C=1.0
Optimizer='GradientDescent'
norm=''
transform_output='tanh'
with_rois_scores_atEnd=False
optim_wt_Reg = False
with_scores=False
epsilon=0.01
restarts_paral='paral'
Max_version=''
w_exp=10.0
seuillage_by_score=False
seuil=0.9
k_intopk=1
C_Searching=False
predict_with='MILSVM'
gridSearch=False
thres_FinalClassifier=0.5
n_jobs=1
thresh_evaluation=0.05
TEST_NMS=0.3
ext = '.txt'
if database=='Paintings':
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/Painting_Dataset/'
classes = ['aeroplane','bird','boat','chair','cow','diningtable','dog','horse','sheep','train']
elif database=='VOC12':
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/VOCdevkit/VOC2012/JPEGImages/'
elif database=='VOC2007':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/VOCdevkit/VOC2007/JPEGImages/'
classes = ['aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat', 'chair',
'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor']
elif database=='watercolor':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/cross-domain-detection/datasets/watercolor/JPEGImages/'
classes = ["bicycle", "bird","car", "cat", "dog", "person"]
elif database=='PeopleArt':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/PeopleArt/JPEGImages/'
classes = ["person"]
elif database=='WikiTenLabels':
ext = '.csv'
item_name = 'item'
path_to_img = '/media/gonthier/HDD/data/Wikidata_Paintings/WikiTenLabels/JPEGImages/'
classes = ['angel', 'beard','capital','Child_Jesus', 'crucifixion_of_Jesus',
'Mary','nudity', 'ruins','Saint_Sebastien','turban']
elif database=='clipart':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/cross-domain-detection/datasets/clipart/JPEGImages/'
classes = ['aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat', 'chair',
'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor']
elif(database=='Wikidata_Paintings'):
item_name = 'image'
path_to_img = '/media/gonthier/HDD/data/Wikidata_Paintings/600/'
raise NotImplemented # TODO implementer cela !!!
elif(database=='Wikidata_Paintings_miniset_verif'):
item_name = 'image'
path_to_img = '/media/gonthier/HDD/data/Wikidata_Paintings/600/'
classes = ['Q235113_verif','Q345_verif','Q10791_verif','Q109607_verif','Q942467_verif']
if testMode and not(type(jtest)==int):
assert(type(jtest)==str)
jtest = int(np.where(np.array(classes)==jtest)[0][0])# Conversion of the jtest string to the value number
assert(type(jtest)==int)
if testMode and (jtest>len(classes)) :
print("We are in test mode but jtest>len(classes), we will use jtest =0" )
jtest =0
path_data = '/media/gonthier/HDD/output_exp/ClassifPaintings/'
databasetxt =path_data + database + ext
if database=='WikiTenLabels':
dtypes = {0:str,'item':str,'angel':int,'beard':int,'capital':int, \
'Child_Jesus':int,'crucifixion_of_Jesus':int,'Mary':int,'nudity':int,'ruins':int,'Saint_Sebastien':int,\
'turban':int,'set':str,'Anno':int}
df_label = pd.read_csv(databasetxt,sep=",",dtype=dtypes)
else:
df_label = pd.read_csv(databasetxt,sep=",")
str_val = 'val'
if database=='Wikidata_Paintings_miniset_verif':
df_label = df_label[df_label['BadPhoto'] <= 0.0]
str_val = 'validation'
elif database=='Paintings':
str_val = 'validation'
elif database in ['VOC2007','watercolor','clipart','PeopleArt']:
str_val = 'val'
df_label[classes] = df_label[classes].apply(lambda x:(x + 1.0)/2.0)
num_trainval_im = len(df_label[df_label['set']=='train'][item_name]) + len(df_label[df_label['set']==str_val][item_name])
num_classes = len(classes)
print(database,'with ',num_trainval_im,' images in the trainval set')
N = 1
extL2 = ''
nms_thresh = 0.7
savedstr = '_all'
thresh_evaluation = 0.05
TEST_NMS =0.3
thresh = 0.05
sets = ['train','val','trainval','test']
dict_name_file = {}
data_precomputeed= True
if k_per_bag==300:
k_per_bag_str = ''
else:
k_per_bag_str = '_k'+str(k_per_bag)
eval_onk300 = True
for set_str in sets:
name_pkl_all_features = path_data+'FasterRCNN_'+ demonet +'_'+database+'_N'+str(N)+extL2+'_TLforMIL_nms_'+str(nms_thresh)+savedstr+k_per_bag_str+'_'+set_str+'.tfrecords'
if not(k_per_bag==300) and eval_onk300 and set_str=='test': # We will evaluate on all the 300 regions and not only the k_per_bag ones
name_pkl_all_features = path_data+'FasterRCNN_'+ demonet +'_'+database+'_N'+str(N)+extL2+'_TLforMIL_nms_'+str(nms_thresh)+savedstr+'_'+set_str+'.tfrecords'
dict_name_file[set_str] = name_pkl_all_features
if not(os.path.isfile(name_pkl_all_features)):
data_precomputeed = False
# sLength_all = len(df_label[item_name])
if demonet == 'vgg16_COCO':
num_features = 4096
elif demonet in ['res101_COCO','res152_COCO','res101_VOC07']:
num_features = 2048
if not(data_precomputeed):
# Compute the features
if verbose: print("We will computer the CNN features")
Compute_Faster_RCNN_features(demonet=demonet,nms_thresh =nms_thresh,
database=database,augmentation=False,L2 =False,
saved='all',verbose=verbose,filesave='tfrecords',k_regions=k_per_bag)
dont_use_07_metric = True
if database=='VOC2007':
imdb = get_imdb('voc_2007_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='watercolor':
imdb = get_imdb('watercolor_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='PeopleArt':
imdb = get_imdb('PeopleArt_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='clipart':
imdb = get_imdb('clipart_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='WikiTenLabels':
imdb = get_imdb('WikiTenLabels_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
#num_images = len(imdb.image_index)
num_images = len(df_label[df_label['set']=='test'][item_name])
else:
num_images = len(df_label[df_label['set']=='test'][item_name])
all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]
data_path_train= dict_name_file['trainval']
# Config param for TF session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Data for the MILSVM Latent SVM
# All those parameter are design for my GPU 1080 Ti memory size
performance = False
if parallel_op:
sizeMax = 30*10000 // (k_per_bag*num_classes)
else:
sizeMax = 30*10000 // k_per_bag
if not(init_by_mean is None) and not(init_by_mean==''):
if not(CV_Mode=='CV' and num_split==2):
sizeMax //= 2
# boolean paralleliation du W
if not(num_features==2048):
sizeMax //= (num_features//2048)
if restarts_paral=='Dim': # It will create a new dimension
restarts_paral_str = '_RP'
sizeMax //= max(int((restarts+1)//2),1) # To avoid division by zero
# it seems that using a different size batch drasticly change the results
elif restarts_paral=='paral': # Version 2 of the parallelisation
restarts_paral_str = '_RPV2'
sizeMax = 30*200000 // (k_per_bag*20)
else:
restarts_paral_str=''
# InternalError: Dst tensor is not initialized. can mean that you are running out of GPU memory
mini_batch_size = min(sizeMax,num_trainval_im)
buffer_size = 10000
if testMode:
ext_test = '_Test_Mode'
else:
ext_test= ''
max_iters = ((num_trainval_im // mini_batch_size)+ \
np.sign(num_trainval_im % mini_batch_size))*max_iters_all_base
AP_per_class = []
P_per_class = []
R_per_class = []
P20_per_class = []
AP_per_classbS = []
final_clf = None
class_weight = None
ReDo = False
if C_Searching:C_Searching_str ='_Csearch'
else: C_Searching_str = ''
if with_scores:
with_scores_str = '_WRC'+str(epsilon)
else:
with_scores_str=''
if seuillage_by_score:
seuillage_by_score_str = '_SBS'+str(seuil)
else: seuillage_by_score_str = ''
if norm=='L2':
extNorm = '_L2'
elif norm=='STDall':
extNorm = '_STDall'
elif norm=='STDSaid':
extNorm = '_STDSaid'
elif norm=='STD':
extNorm = '_STD'
raise(NotImplemented)
elif norm=='' or norm is None:
extNorm = ''
if parallel_op:
extPar = '_p'
else:
extPar =''
if CV_Mode=='CV':
max_iters = (max_iters*(num_split-1)//num_split) # Modification d iteration max par rapport au nombre de split
extCV = '_cv'+str(num_split)
elif CV_Mode=='LA':
max_iters = (max_iters*(num_split-1)//num_split) # Modification d iteration max par rapport au nombre de split
extCV = '_la'+str(num_split)
elif CV_Mode=='CV' and WR==True:
extCV = '_cv'+str(num_split)+'_wr'
elif CV_Mode is None or CV_Mode=='':
extCV =''
if WR: extCV += '_wr'
if Optimizer=='Adam':
opti_str=''
elif Optimizer=='GradientDescent':
opti_str='_gd'
else:
raise(NotImplemented)
if init_by_mean is None or init_by_mean=='':
init_by_mean_str = ''
elif init_by_mean=='First':
init_by_mean_str= '_ibnF'
elif init_by_mean=='All':
init_by_mean_str= '_ibnA'
if LR==0.01:
LR_str = ''
else:
LR_str='_LR'+str(LR)
if C == 1.0:
C_str=''
else:
C_str = '_C'+str(C) # regularisation term
if Max_version=='max' or Max_version=='' or Max_version is None:
Max_version_str =''
elif Max_version=='softmax':
Max_version_str ='_MVSF'
if not(w_exp==1.0): Max_version_str+=str(w_exp)
elif Max_version=='sparsemax':
Max_version_str ='_MVSM'
elif Max_version=='mintopk':
Max_version_str ='_MVMT'+str(k_intopk)
optimArg= None
verboseMILSVM = True
shuffle = True
if num_trainval_im==mini_batch_size:
shuffle = False
if shuffle:
shuffle_str = ''
else:
shuffle_str = '_allBase'
if optim_wt_Reg:
optim_wt_Reg_str = '_OptimWTReg'
else:
optim_wt_Reg_str =''
Number_of_positif_elt = 1
number_zone = k_per_bag
# thresh_evaluation,TEST_NMS = 0.05,0.3
dont_use_07_metric = True
symway = True
arrayParam = [demonet,database,N,extL2,nms_thresh,savedstr,mini_batch_size,
performance,buffer_size,predict_with,shuffle,C,testMode,restarts,max_iters_all_base,
max_iters,CV_Mode,num_split,parallel_op,WR,norm,Optimizer,LR,optimArg,
Number_of_positif_elt,number_zone,seuil_estimation,thresh_evaluation,
TEST_NMS,init_by_mean,transform_output,with_rois_scores_atEnd,
with_scores,epsilon,restarts_paral,Max_version,w_exp,seuillage_by_score,seuil,
k_intopk,C_Searching,gridSearch,thres_FinalClassifier,optim_wt_Reg]
arrayParamStr = ['demonet','database','N','extL2','nms_thresh','savedstr',
'mini_batch_size','performance','buffer_size','predict_with',
'shuffle','C','testMode','restarts','max_iters_all_base','max_iters','CV_Mode',
'num_split','parallel_op','WR','norm','Optimizer','LR',
'optimArg','Number_of_positif_elt','number_zone','seuil_estimation'
,'thresh_evaluation','TEST_NMS','init_by_mean','transform_output','with_rois_scores_atEnd',
'with_scores','epsilon','restarts_paral','Max_version','w_exp','seuillage_by_score',
'seuil','k_intopk','C_Searching','gridSearch','thres_FinalClassifier','optim_wt_Reg']
assert(len(arrayParam)==len(arrayParamStr))
print(tabs_to_str(arrayParam,arrayParamStr))
# print('database',database,'mini_batch_size',mini_batch_size,'max_iters',max_iters,'norm',norm,\
# 'parallel_op',parallel_op,'CV_Mode',CV_Mode,'WR',WR,'restarts',restarts,'demonet',demonet,
# 'Optimizer',Optimizer,'init_by_mean',init_by_mean,'with_tanh',with_tanh)
cachefile_model_base= database +'_'+demonet+'_r'+str(restarts)+'_s' \
+str(mini_batch_size)+'_k'+str(k_per_bag)+'_m'+str(max_iters)+extNorm+extPar+\
extCV+ext_test+opti_str+LR_str+C_str+init_by_mean_str+with_scores_str+restarts_paral_str\
+Max_version_str+seuillage_by_score_str+shuffle_str+C_Searching_str+optim_wt_Reg_str
cachefile_model = path_data + cachefile_model_base+'_MILSVM.pkl'
with open(cachefile_model, 'rb') as f:
name_milsvm = pickle.load(f)
export_dir,np_pos_value,np_neg_value= name_milsvm
export_dir_path = ('/').join(export_dir.split('/')[:-1])
name_model_meta = export_dir + '.meta'
get_roisScore = False
train_dataset = tf.data.TFRecordDataset(dict_name_file['test'])
if not(k_per_bag==300) and eval_onk300:
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore,num_features=num_features,num_rois=300))
else:
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore,num_features=num_features,num_rois=k_per_bag))
dataset_batch = train_dataset.batch(mini_batch_size)
dataset_batch.cache()
iterator = dataset_batch.make_one_shot_iterator()
next_element = iterator.get_next()
true_label_all_test = []
predict_label_all_test = []
name_all_test = []
FirstTime= True
i = 0
ii = 0
load_model = False
scoreInMILSVM =False
seuil_estimation_bool = False
with_rois_scores_atEnd=False
seuillage_by_score=False
gridSearch=False
with_softmax_a_intraining= False
with_softmax = False
sess = tf.Session(config=config)
if load_model==False:
new_saver = tf.train.import_meta_graph(name_model_meta)
new_saver.restore(sess, tf.train.latest_checkpoint(export_dir_path))
graph= tf.get_default_graph()
X = graph.get_tensor_by_name("X:0")
y = graph.get_tensor_by_name("y:0")
if scoreInMILSVM:
scores_tf = graph.get_tensor_by_name("scores:0")
Prod_best = graph.get_tensor_by_name("ProdScore:0")
else:
Prod_best = graph.get_tensor_by_name("Prod:0")
if with_tanh:
print('use of tanh')
Tanh = tf.tanh(Prod_best)
mei = tf.argmax(Tanh,axis=2)
score_mei = tf.reduce_max(Tanh,axis=2)
elif with_softmax:
Softmax = tf.nn.softmax(Prod_best,axis=-1)
mei = tf.argmax(Softmax,axis=2)
score_mei = tf.reduce_max(Softmax,axis=2)
elif with_softmax_a_intraining:
Softmax=tf.multiply(tf.nn.softmax(Prod_best,axis=-1),Prod_best)
mei = tf.argmax(Softmax,axis=2)
score_mei = tf.reduce_max(Softmax,axis=2)
else:
mei = tf.argmax(Prod_best,axis=2)
score_mei = tf.reduce_max(Prod_best,axis=2)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
while True:
try:
if not(with_rois_scores_atEnd) and not(scoreInMILSVM):
fc7s,roiss, labels,name_imgs = sess.run(next_element)
else:
fc7s,roiss,rois_scores,labels,name_imgs = sess.run(next_element)
fc7s_full,roiss_full, labels_full,name_imgs_full = fc7s,roiss, labels,name_imgs
PositiveExScoreAll_tab = []
for iii in range(10):
fc7s = fc7s_full[:,iii*30:iii*30+30,:]
roiss = roiss_full[:,iii*30:iii*30+30]
labels = labels_full
name_imgs = name_imgs_full
if predict_with=='MILSVM':
if scoreInMILSVM:
feed_dict_value = {X: fc7s,scores_tf: rois_scores, y: labels}
else:
feed_dict_value = {X: fc7s, y: labels}
if with_tanh:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Tanh], feed_dict=feed_dict_value)
elif with_softmax or with_softmax_a_intraining:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Softmax], feed_dict=feed_dict_value)
else:
PositiveRegions,get_RegionsScore,PositiveExScoreAll = \
sess.run([mei,score_mei,Prod_best], feed_dict=feed_dict_value)
if with_rois_scores_atEnd:
PositiveExScoreAll = PositiveExScoreAll*rois_scores
get_RegionsScore = np.max(PositiveExScoreAll,axis=2)
PositiveRegions = np.amax(PositiveExScoreAll,axis=2)
if with_tanh: assert(np.max(PositiveExScoreAll) <= 1.)
PositiveExScoreAll_tab += [PositiveExScoreAll]
roiss = roiss_full
PositiveExScoreAll= np.concatenate(PositiveExScoreAll_tab,axis=2)
PositiveRegions = np.amax(PositiveExScoreAll,axis=2)
get_RegionsScore= np.max(PositiveExScoreAll,axis=2)
print('PositiveExScoreAll',PositiveExScoreAll.shape)
print('PositiveRegions',PositiveRegions.shape)
print('get_RegionsScore',get_RegionsScore.shape)
print('name_imgs',len(name_imgs))
true_label_all_test += [labels]
if predict_with=='MILSVM':
predict_label_all_test += [get_RegionsScore]
elif 'LinearSVC' in predict_with:
predict_label_all_test_tmp = []
for j in range(num_classes):
predict_label_all_test_tmp += [np.reshape(classifier_trained_dict[j].decision_function(
np.reshape(fc7s,(-1,fc7s.shape[-1]))),(fc7s.shape[0],fc7s.shape[1]))]
predict_label_all_test_batch = np.stack(predict_label_all_test_tmp,axis=0)
# print(predict_label_all_test_batch.shape)
predict_label_all_test += [np.max(predict_label_all_test_batch,axis=2)]
# print('predict_label_all_test',predict_label_all_test[-1].shape)
# predict_label_all_test is used only for the classification score !
# if predict_with=='LinearSVC':
for k in range(len(labels)):
if database in ['VOC2007','watercolor','Paintings','clipart','WikiTenLabels','PeopleArt'] :
complet_name = path_to_img + str(name_imgs[k].decode("utf-8")) + '.jpg'
else:
complet_name = path_to_img + name_imgs[k] + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
if predict_with=='MILSVM':
scores_all = PositiveExScoreAll[:,k,:]
elif 'LinearSVC' in predict_with:
scores_all = predict_label_all_test_batch[:,k,:]
roi = roiss[k,:]
roi_boxes = roi[:,1:5] / im_scales[0]
for j in range(num_classes):
scores = scores_all[j,:]
if seuil_estimation_bool:
inds = np.where(scores > list_thresh[j])[0]
else:
inds = np.where(scores > thresh)[0]
cls_scores = scores[inds]
cls_boxes = roi_boxes[inds,:]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32, copy=False)
modif_box = '' # Possibility : SumPond, Inter
if(not(modif_box=='') and not(modif_box is None)):
# Modification of the bounding box
cls_dets = py_cpu_modif(cls_dets,kind=modif_box)
keep = nms(cls_dets, TEST_NMS)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
i+=1
for l in range(len(name_imgs)):
if database in ['VOC2007','watercolor','clipart','WikiTenLabels','PeopleArt']:
name_all_test += [[str(name_imgs[l].decode("utf-8"))]]
else:
name_all_test += [[name_imgs[l]]]
if PlotRegions and predict_with=='MILSVM':
if verbose and (ii%1000==0):
print("Plot the images :",ii)
if verbose and FirstTime:
FirstTime = False
print("Start ploting Regions on test set")
for k in range(len(labels)):
if ii > number_im:
continue
if database in ['VOC2007','Paintings','watercolor','clipart','WikiTenLabels','PeopleArt']:
name_img = str(name_imgs[k].decode("utf-8") )
else:
name_img = name_imgs[k]
rois = roiss[k,:]
if database in ['VOC12','Paintings','VOC2007','clipart','watercolor','WikiTenLabels','PeopleArt']:
complet_name = path_to_img + name_img + '.jpg'
name_sans_ext = name_img
elif(database=='Wikidata_Paintings') or (database=='Wikidata_Paintings_miniset_verif'):
name_sans_ext = os.path.splitext(name_img)[0]
complet_name = path_to_img +name_sans_ext + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
roi_boxes_and_score = []
local_cls = []
for j in range(num_classes):
cls_dets = all_boxes[j][ii] # Here we have #classe x box dim + score
# Last element is the score
# print(cls_dets.shape)
if len(cls_dets) > 0:
local_cls += [classes[j]]
# roi_boxes_score = np.expand_dims(cls_dets,axis=0)
roi_boxes_score = cls_dets
# print(roi_boxes_score.shape)
if roi_boxes_and_score is None:
roi_boxes_and_score = [roi_boxes_score]
else:
roi_boxes_and_score += [roi_boxes_score]
#np.vstack((roi_boxes_and_score,roi_boxes_score))
if roi_boxes_and_score is None: roi_boxes_and_score = [[]]
ii += 1
if RPN:
best_RPN_roi = rois[0,:]
best_RPN_roi_boxes = best_RPN_roi[1:5] / im_scales[0]
best_RPN_roi_scores = [PositiveExScoreAll[j,k,0]]
cls = local_cls + ['RPN'] # Comparison of the best region according to the faster RCNN and according to the MILSVM de Said
#best_RPN_roi_boxes_score = np.expand_dims(np.expand_dims(np.concatenate((best_RPN_roi_boxes,best_RPN_roi_scores)),axis=0),axis=0)
best_RPN_roi_boxes_score = np.expand_dims(np.concatenate((best_RPN_roi_boxes,best_RPN_roi_scores)),axis=0)
# print(best_RPN_roi_boxes_score.shape)
#roi_boxes_and_score = np.vstack((roi_boxes_and_score,best_RPN_roi_boxes_score))
roi_boxes_and_score += [best_RPN_roi_boxes_score] #np.vstack((roi_boxes_and_score,best_RPN_roi_boxes_score))
else:
cls = local_cls
#print(len(cls),len(roi_boxes_and_score))
vis_detections_list(im, cls, roi_boxes_and_score, thresh=-np.inf)
name_output = path_to_output2 +'Test/' + name_sans_ext + '_Regions.jpg'
if database=='PeopleArt':
path_tmp = '/'.join(name_output.split('/')[0:-1])
pathlib.Path(path_tmp).mkdir(parents=True, exist_ok=True)
plt.savefig(name_output)
plt.close()
except tf.errors.OutOfRangeError:
break
print('End compute')
tf.reset_default_graph()
true_label_all_test = np.concatenate(true_label_all_test)
predict_label_all_test = np.transpose(np.concatenate(predict_label_all_test,axis=1))
name_all_test = np.concatenate(name_all_test)
labels_test_predited = (np.sign(predict_label_all_test) +1.)/2
labels_test_predited[np.where(labels_test_predited==0.5)] = 0 # To deal with the case where predict_label_all_test == 0
for j,classe in enumerate(classes):
AP = average_precision_score(true_label_all_test[:,j],predict_label_all_test[:,j],average=None)
print("MIL-SVM version Average Precision for",classes[j]," = ",AP)
test_precision = precision_score(true_label_all_test[:,j],labels_test_predited[:,j],)
test_recall = recall_score(true_label_all_test[:,j],labels_test_predited[:,j],)
F1 = f1_score(true_label_all_test[:,j],labels_test_predited[:,j],)
print("Test on all the data precision = {0:.2f}, recall = {1:.2f},F1 = {2:.2f}".format(test_precision,test_recall,F1))
precision_at_k = ranking_precision_score(np.array(true_label_all_test[:,j]), predict_label_all_test[:,j],20)
P20_per_class += [precision_at_k]
AP_per_class += [AP]
R_per_class += [test_recall]
P_per_class += [test_precision]
det_file = os.path.join(path_data, 'detections_aux.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL)
max_per_image = 100
num_images_detect = len(imdb.image_index) # We do not have the same number of images in the WikiTenLabels case
all_boxes_order = [[[] for _ in range(num_images_detect)] for _ in range(imdb.num_classes)]
number_im = 0
for i in range(num_images_detect):
name_img = imdb.image_path_at(i)
if database=='PeopleArt':
name_img_wt_ext = name_img.split('/')[-2] +'/' +name_img.split('/')[-1]
name_img_wt_ext_tab =name_img_wt_ext.split('.')
name_img_wt_ext = '.'.join(name_img_wt_ext_tab[0:-1])
else:
name_img_wt_ext = name_img.split('/')[-1]
name_img_wt_ext =name_img_wt_ext.split('.')[0]
name_img_ind = np.where(np.array(name_all_test)==name_img_wt_ext)[0]
#print(name_img_ind)
if len(name_img_ind)==0:
print('len(name_img_ind), images not found in the all_boxes')
print(name_img_wt_ext)
raise(Exception)
else:
number_im += 1
#print(name_img_ind[0])
for j in range(1, imdb.num_classes):
j_minus_1 = j-1
all_boxes_order[j][i] = all_boxes[j_minus_1][name_img_ind[0]]
if max_per_image > 0:
image_scores = np.hstack([all_boxes_order[j][i][:, -1]
for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes_order[j][i][:, -1] >= image_thresh)[0]
all_boxes_order[j][i] = all_boxes_order[j][i][keep, :]
assert (number_im==num_images_detect) # To check that we have the all the images in the detection prediction
det_file = os.path.join(path_data, 'detections.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data +'tmp/' + database+'_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
print("Detection score (thres = 0.5): ",database)
print(arrayToLatex(aps,per=True))
ovthresh_tab = [0.3,0.1,0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order, output_dir,ovthresh)
print("Detection score with thres at ",ovthresh)
print(arrayToLatex(aps,per=True))
imdb.set_use_diff(True) # Modification of the use_diff attribute in the imdb
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
print("Detection score with the difficult element")
print(arrayToLatex(aps,per=True))
imdb.set_use_diff(False)
print('~~~~~~~~')
print("mean Average Precision Classification for all the data = {0:.3f}".format(np.mean(AP_per_class)))
if CompBest: print("mean Average Precision for BEst Score = {0:.3f}".format(np.mean(AP_per_classbS)))
print("mean Precision Classification for all the data = {0:.3f}".format(np.mean(P_per_class)))
print("mean Recall Classification for all the data = {0:.3f}".format(np.mean(R_per_class)))
print("mean Precision Classification @ 20 for all the data = {0:.3f}".format(np.mean(P20_per_class)))
print('Mean Average Precision Classification :')
print(AP_per_class)
print(arrayToLatex(AP_per_class,per=True))
return(0)