def eval_MAP_SaliencyMethods(database='IconArt_v1',metamodel='FasterRCNN',demonet='res152_COCO',
k_per_bag=300,SaliencyMethod='SmoothGrad'):
"""
The goal of this function is to compute the mAP of the saliency method for
classification ResNet
@param : SaliencyMethod : IntegratedGrad ou SmoothGrad pour le moment
"""
matplotlib.use('Agg')
save_data = False
ReDo = True
plot = False
TEST_NMS = 0.01
thresh_classif = 0.1
# Parameter for the classification network
target_dataset = 'IconArt_v1'
style_layers = []
features = 'activation_48'
normalisation = False
final_clf= 'LinearSVC' # Don t matter
source_dataset= 'ImageNet'
transformOnFinalLayer='GlobalAveragePooling2D'
final_clf = 'MLP2'
epochs = 20
optimizer = 'SGD'
return_best_model = True
batch_size= 16
dropout=None
regulOnNewLayer=None
nesterov=False
SGDmomentum=0.9
decay=1e-4
cropCenter = False
# Load ResNet50 normalisation statistics
opt_option = [0.1,0.01]
pretrainingModif = True
kind_method = 'FT'
computeGlobalVariance = False
constrNet = 'ResNet50'
sizeIm = 224
Net = constrNet
# Load the box proosals
item_name,path_to_img,default_path_imdb,classes,ext,num_classes,str_val,df_label,path_data,Not_on_NicolasPC = get_database(database)
imdb,list_im_withanno = get_imdb_and_listImagesInTestSet(database)
num_images_detect = len(list_im_withanno)
dict_rois = getDictBoxesProposals(database=target_dataset,k_per_bag=k_per_bag,\
metamodel=metamodel,demonet=demonet)
for_data_output = os.path.join(path_data,'dataSaliencyMap',SaliencyMethod)
im_with_boxes_output = os.path.join(path_data,'SaliencyMapImagesBoxes',SaliencyMethod)
print('===',im_with_boxes_output)
pathlib.Path(for_data_output).mkdir(parents=True, exist_ok=True)
pathlib.Path(im_with_boxes_output).mkdir(parents=True, exist_ok=True)
# Load Classification model
print('loading :',constrNet,computeGlobalVariance,kind_method,pretrainingModif,opt_option)
model = learn_and_eval(target_dataset,source_dataset,final_clf,features,\
constrNet,kind_method,style_layers=style_layers,
normalisation=normalisation,transformOnFinalLayer=transformOnFinalLayer,
batch_size_RF=16,epochs_RF=20,momentum=0.9,ReDo=False,
returnStatistics=True,cropCenter=cropCenter,\
computeGlobalVariance=computeGlobalVariance,\
epochs=epochs,optimizer=optimizer,opt_option=opt_option,
return_best_model=return_best_model,\
batch_size=batch_size,gridSearch=False,verbose=True)
SaliencyMapClass_tab = []
stdev_spread = 0.1
nsamples = 50
x_steps = 50
for j in range(num_classes):
SaliencyMapClass=getSaliencyMapClass(model,c_i=j,method=SaliencyMethod,\
stdev_spread=stdev_spread,nsamples=nsamples,x_steps=x_steps)
SaliencyMapClass_tab +=[SaliencyMapClass]
# list_gt_boxes_classes = []
candidate_boxes = [[] for _ in range(imdb.num_images)]
all_boxes_order = [[[] for _ in range(num_images_detect)] for _ in range(imdb.num_classes)]
# number_gt_boxes = 0
itera = 20
norm = True
t0 = time.time()
# Un peu plus de 1440 images
for i in range(imdb.num_images):
# complet_name_tab = ('.'.join(complet_name.split('.')[0:-1])).split('/')
im_path = imdb.image_path_at(i)
name_im = im_path.split('/')[-1]
if i%itera==0:
t1 = time.time()
print(i,name_im,'duration for ',itera,'iterations = ',str(t1-t0),'s')
t0 = time.time()
im = cv2.imread(im_path)
hauteur, largeur ,_ = im.shape
blobs, im_scales = get_blobs(im)
if database=='PeopleArt':
name_im = '/'.join(im_path.split('/')[-2:])
if database=='PeopleArt':
name_im= '.'.join(name_im.split('.')[0:-1])
else:
name_im = name_im.split('.')[0]
proposals_boxes = dict_rois[name_im]
if cropCenter:
image_array= load_and_crop_img(path=im_path,Net=Net,target_size=sizeIm,
crop_size=sizeIm,interpolation='lanczos:center')
# For VGG or ResNet with classification head size == 224
else:
image_array = load_resize_and_process_img(im_path,Net=Net,max_dim=sizeIm)
#print(np.max(image_array),np.min(image_array),np.mean(image_array),np.median(image_array))
#input('wait')
dict_sensitivity = {}
dict_sensitivity_path = os.path.join(for_data_output,name_im+'_dict_SaliencyMap'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+'.pkl')
if not(os.path.exists(dict_sensitivity_path)) or ReDo:
predictions = model.predict(image_array)[0]
dict_sensitivity['predictions'] = predictions
inds = np.where(predictions > thresh_classif)[0]
for ind in inds:
prediction = predictions[ind]
if np.isnan(prediction):
print('Prediction of ',name_im,'is nan !!!')
input('wait')
candidate_boxes = []
j = ind +1 # the class index for the evaluation part
Smap=SaliencyMapClass_tab[ind].GetMask(image_array)
#print('before normalisation',np.max(Smap),np.min(Smap),np.mean(Smap),np.median(Smap))
if save_data: dict_sensitivity[j] = Smap
if SaliencyMethod=='SmoothGrad':
#Smap_grey = np.mean(Smap,axis=-1,keepdims=True)
Smap_grey = np.mean(np.abs(Smap),axis=-1,keepdims=True)
#print('after grey',np.max(Smap_grey),np.min(Smap_grey),np.mean(Smap_grey),np.median(Smap_grey))
if norm:
Smap_grey = to_01(Smap_grey)
#print('after normalisation',np.max(Smap_grey),np.min(Smap_grey),np.mean(Smap_grey),np.median(Smap_grey))
Smap_grey_time_score = prediction*Smap_grey
else: # In the case of Integrated Gradient
# Sur conseil d Antoine Pirovano
ptile= 99
# Sum for grayscale of the absolute value
pixel_attrs = np.sum(np.abs(Smap), axis=-1,keepdims=True)
pixel_attrs = np.clip(pixel_attrs / np.percentile(pixel_attrs, ptile), 0, 1)
Smap_grey_time_score = prediction * pixel_attrs
#print('after mul score',np.max(Smap_grey_time_score),np.min(Smap_grey_time_score),np.mean(Smap_grey_time_score),np.median(Smap_grey_time_score))
# attention truc super contre intuitif dans le resize c'est hauteur largeur alors que
# la fonction size retourne largeur hauteur
Smap_grey_time_score = Smap_grey_time_score[0]
#Smap_grey_time_score_resized = cv2.resize(Smap_grey_time_score, (hauteur, largeur),interpolation=cv2.INTER_NEAREST)
Smap_grey_time_score_resized = cv2.resize(Smap_grey_time_score, (largeur,hauteur),interpolation=cv2.INTER_NEAREST)
#print('Smap_grey_time_score_resized',Smap_grey_time_score_resized.shape,im.shape)
#print('after resize',np.max(Smap_grey_time_score_resized),np.min(Smap_grey_time_score_resized),np.mean(Smap_grey_time_score_resized),np.median(Smap_grey_time_score_resized))
if plot:
name_output = name_im+'_'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+ '_'+str(j)+'.jpg'
name_output_path = os.path.join(im_with_boxes_output,name_output)
Smap_grey_time_score_resized_01 = to_01(Smap_grey_time_score_resized)
plt.imshow(Smap_grey_time_score_resized_01, cmap=cm.gray)
plt.title(classes[j-1]+' : '+str(prediction))
plt.savefig(name_output_path)
plt.close()
for k in range(len(proposals_boxes)):
box = proposals_boxes[k]
x1,y1,x2,y2 = box # x : largeur et y en hauteur
x1_int = int(np.round(x1))
x2_int = int(np.round(x2))
y1_int = int(np.round(y1))
y2_int = int(np.round(y2))
#print(name_im,'Smap_grey_time_score_resized',Smap_grey_time_score_resized.shape,im.shape)
#print(x1_int,x2_int,y1_int,y2_int)
assert(x2_int<=largeur)
assert(y2_int<=hauteur)
Smap_grey_time_score_resized_crop = Smap_grey_time_score_resized[y1_int:y2_int,x1_int:x2_int]
# because bbox = dets[i, :4] # Boxes are score, x1,y1,x2,y2
Smap_grey_time_score_resized_crop_score = np.mean(Smap_grey_time_score_resized_crop)
# if k < 3:
# print('Smap_grey_time_score_resized_crop',Smap_grey_time_score_resized_crop.shape)
# print(x1_int,x2_int,y1_int,y2_int)
# print('j',j,'k',k,',score',Smap_grey_time_score_resized_crop_score)
if not(np.isnan(Smap_grey_time_score_resized_crop_score)):
box_with_scores = np.append(box,[Smap_grey_time_score_resized_crop_score])
candidate_boxes += [box_with_scores]
else:
box_with_scores = np.append(box,[0.0])
candidate_boxes += [box_with_scores]
# if np.isnan(Smap_grey_time_score_resized_crop_score):
# print('!!! score is nan')
# print(x1,y1,x2,y2)
# print(x1_int,x2_int,y1_int,y2_int)
# print(Smap_grey_time_score_resized_crop.shape)
# print(name_im,'Smap_grey_time_score_resized',Smap_grey_time_score_resized.shape,im.shape)
# print(prediction)
# print('after resize',np.max(Smap_grey_time_score_resized),np.min(Smap_grey_time_score_resized),np.mean(Smap_grey_time_score_resized),np.median(Smap_grey_time_score_resized))
# print(Smap_grey_time_score_resized_crop_score)
# input('wait')
#print(candidate_boxes)
if len(candidate_boxes)>0:
candidate_boxes_NP = np.array(candidate_boxes)
candidate_boxes_NP[:,-1] = candidate_boxes_NP[:,-1] -np.max(candidate_boxes_NP[:,-1]) + prediction
keep = nms(candidate_boxes_NP, TEST_NMS)
cls_dets = candidate_boxes_NP[keep, :]
all_boxes_order[j][i] = cls_dets
if plot:
roi_boxes_and_score = []
local_cls = []
for j in range(num_classes):
cls_dets = all_boxes_order[j+1][i]
if len(cls_dets) > 0:
local_cls += [classes[j]]
roi_boxes_score = cls_dets
if roi_boxes_and_score is None:
roi_boxes_and_score = [roi_boxes_score]
else:
roi_boxes_and_score += [roi_boxes_score]
if roi_boxes_and_score is None: roi_boxes_and_score = [[]]
#print(name_im,roi_boxes_and_score,local_cls)
vis_detections_list(im, local_cls, roi_boxes_and_score, thresh=-np.inf)
name_output = name_im+'_'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+ '_Regions.jpg'
name_output_path = os.path.join(im_with_boxes_output,name_output)
#input("wait")
plt.savefig(name_output_path)
plt.close()
if save_data:
with open(dict_sensitivity_path, 'wb') as f:
pickle.dump(dict_sensitivity, f, pickle.HIGHEST_PROTOCOL)
# for i in range(imdb.num_images):
# candidate_boxes[i] = np.array(candidate_boxes[i])
imdb.set_force_dont_use_07_metric(True)
det_file = os.path.join(path_data, 'detectionsSaliencyMap'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+'.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data +'tmp/' + database+'_'+SaliencyMethod+'_std'+str(stdev_spread)+'_n'+str(nsamples)+'_steps'+str(x_steps)+'_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir) # AP at O.5
print("===> Detection score (thres = 0.5): ",database,'with Saliency map from',SaliencyMethod,'with std =',stdev_spread,'nsamples = ',nsamples,'x_steps =',x_steps)
print(arrayToLatex(aps,per=True))
ovthresh_tab = [0.3,0.1,0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order, output_dir,ovthresh)
print("Detection score with thres at ",ovthresh)
print(arrayToLatex(aps,per=True))
def mainEval(dataset_nm='IconArt_v1',classe=0,k_per_bag = 300,metamodel = 'FasterRCNN',\
demonet='res152_COCO',test=False,MILmodel='MI_Net',max_epoch=20,verbose=True):
# dataset_nm='IconArt_v1'
# classe=1
# k_per_bag = 300
# metamodel = 'FasterRCNN'
# demonet='res152_COCO'
# test=True
# MILmodel='MI_Net_with_DS'
# max_epoch = 1
t0 = time.time()
if test:
classe = 0
if MILmodel=='MI_Net':
MILmodel_fct = MI_Net_WSOD
elif MILmodel=='MI_Max_AddOneLayer_Keras':
MILmodel_fct = MI_Max_AddOneLayer_Keras
elif MILmodel=='mi_Net':
MILmodel_fct = mi_Net_WSOD
elif MILmodel=='MI_Net_with_DS':
MILmodel_fct = MI_Net_with_DS_WSOD
elif MILmodel=='MI_Net_with_RC':
MILmodel_fct = MI_Net_with_RC_WSOD
else:
print(MILmodel,'is unkwon')
return(0)
print('MILmodel',MILmodel,max_epoch)
item_name,path_to_img,default_path_imdb,classes,ext,num_classes,str_val,df_label,\
path_data,Not_on_NicolasPC = get_database(dataset_nm)
dataset,bags_full_label,mean_fea,std_fea = load_dataset(dataset_nm,classe=0,k_per_bag = k_per_bag,metamodel = metamodel,demonet=demonet)
model_dict = {}
for j in range(num_classes):
if test and not(j==classe):
continue
else:
for k in range(len(dataset['train'])):
a = list(dataset['train'][k])
a[1] = [bags_full_label[k,j]]*k_per_bag
a = tuple(a)
dataset['train'][k] = a
print('start training for class',j)
model = MILmodel_fct(dataset,max_epoch=max_epoch,verbose=verbose)
model_dict[j] = model
t1 = time.time()
print("--- Training duration :",str(t1-t0),' s')
dict_name_file = getDictFeaturesPrecomputed(dataset_nm,k_per_bag=k_per_bag,\
metamodel=metamodel,demonet=demonet)
name_file = dict_name_file['test']
if metamodel=='EdgeBoxes':
dim_rois = 4
else:
dim_rois = 5
next_element = getTFRecordDataset(name_file,k_per_bag =k_per_bag,dim_rois = dim_rois,
num_classes = num_classes)
dont_use_07_metric = False
if dataset_nm=='VOC2007':
imdb = get_imdb('voc_2007_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='watercolor':
imdb = get_imdb('watercolor_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='PeopleArt':
imdb = get_imdb('PeopleArt_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='clipart':
imdb = get_imdb('clipart_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='comic':
imdb = get_imdb('comic_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='CASPApaintings':
imdb = get_imdb('CASPApaintings_test',data_path=default_path_imdb)
num_images = len(imdb.image_index)
elif dataset_nm=='IconArt_v1' or dataset_nm=='RMN':
imdb = get_imdb('IconArt_v1_test',data_path=default_path_imdb)
num_images = len(df_label[df_label['set']=='test'][item_name])
elif 'IconArt_v1' in dataset_nm and not('IconArt_v1' ==dataset_nm):
imdb = get_imdb('IconArt_v1_test',ext=dataset_nm.split('_')[-1],data_path=default_path_imdb)
# num_images = len(imdb.image_index)
num_images = len(df_label[df_label['set']=='test'][item_name])
elif dataset_nm in ['WikiTenLabels','MiniTrain_WikiTenLabels','WikiLabels1000training']:
imdb = get_imdb('WikiTenLabels_test',data_path=default_path_imdb)
#num_images = len(imdb.image_index)
num_images = len(df_label[df_label['set']=='test'][item_name])
elif 'OIV5' in dataset_nm: # For OIV5 for instance !
num_images = len(df_label[df_label['set']=='test'][item_name])
else:
num_images = len(df_label[df_label['set']=='test'][item_name])
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]
TEST_NMS = 0.3
thresh = 0.0
true_label_all_test = []
predict_label_all_test = []
name_all_test = []
config = tf.ConfigProto()
config.intra_op_parallelism_threads = 16
config.inter_op_parallelism_threads = 16
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
i = 0
num_features = 2048
while True:
try:
fc7s,roiss,rois_scores,labels,name_imgs = sess.run(next_element)
fc7s = np.divide(fc7s-mean_fea, std_fea).astype(np.float32)
true_label_all_test += [labels]
score_all = None
for j in range(num_classes):
if not(test):
model= model_dict[j]
predictions = model.predict(fc7s.reshape((-1,num_features)),batch_size=1)
if MILmodel=='MI_Net_with_DS':
predictions = predictions[3]
scores_all_j_k = predictions.reshape((fc7s.shape[0],1,fc7s.shape[1]))
else:
if j==classe:
model= model_dict[j]
predictions = model.predict(fc7s.reshape((-1,num_features)),batch_size=1)
if MILmodel=='MI_Net_with_DS':
predictions = predictions[3]
scores_all_j_k = predictions.reshape((fc7s.shape[0],1,fc7s.shape[1]))
if score_all is None:
score_all = scores_all_j_k
else:
score_all = np.concatenate((score_all,scores_all_j_k),axis=1)
predict_label_all_test += [np.max(score_all,axis=2)]
for k in range(len(labels)):
name_im = name_imgs[k].decode("utf-8")
complet_name = path_to_img + str(name_im) + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
roi = roiss[k,:]
if metamodel=='EdgeBoxes':
roi_boxes = roi / im_scales[0]
else:
roi_boxes = roi[:,1:5] / im_scales[0]
for j in range(num_classes):
scores = score_all[k,j,:]
#print(j,'scores',scores.shape)
inds = np.where(scores > thresh)[0]
cls_scores = scores[inds]
cls_boxes = roi_boxes[inds,:]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32, copy=False)
keep = nms(cls_dets, TEST_NMS)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
i += 1
for l in range(len(name_imgs)):
if dataset_nm in ['IconArt_v1','VOC2007','watercolor','clipart',\
'comic','CASPApaintings','WikiTenLabels','PeopleArt',\
'MiniTrain_WikiTenLabels','WikiLabels1000training']:
name_all_test += [[str(name_imgs[l].decode("utf-8"))]]
else:
name_all_test += [[name_imgs[l]]]
except tf.errors.OutOfRangeError:
break
sess.close()
true_label_all_test = np.concatenate(true_label_all_test)
predict_label_all_test = np.concatenate(predict_label_all_test,axis=0)
name_all_test = np.concatenate(name_all_test)
AP_per_class = []
for j,classe in enumerate(classes):
AP = average_precision_score(true_label_all_test[:,j],predict_label_all_test[:,j],average=None)
AP_per_class += [AP]
print('Average Precision classification task :')
print(arrayToLatex(AP_per_class,per=True))
max_per_image = 100
num_images_detect = len(imdb.image_index) # We do not have the same number of images in the WikiTenLabels or IconArt_v1 case
all_boxes_order = [[[] for _ in range(num_images_detect)] for _ in range(imdb.num_classes)]
number_im = 0
name_all_test = name_all_test.astype(str)
for i in range(num_images_detect):
# print(i)
name_img = imdb.image_path_at(i)
if dataset_nm=='PeopleArt':
name_img_wt_ext = name_img.split('/')[-2] +'/' +name_img.split('/')[-1]
name_img_wt_ext_tab =name_img_wt_ext.split('.')
name_img_wt_ext = '.'.join(name_img_wt_ext_tab[0:-1])
else:
name_img_wt_ext = name_img.split('/')[-1]
name_img_wt_ext =name_img_wt_ext.split('.')[0]
name_img_ind = np.where(np.array(name_all_test)==name_img_wt_ext)[0]
#print(name_img_ind)
if len(name_img_ind)==0:
print('len(name_img_ind), images not found in the all_boxes')
print(name_img_wt_ext)
raise(Exception)
else:
number_im += 1
# print(name_img_ind[0])
for j in range(1, imdb.num_classes):
j_minus_1 = j-1
if len(all_boxes[j_minus_1][name_img_ind[0]]) >0:
all_boxes_order[j][i] = all_boxes[j_minus_1][name_img_ind[0]]
if max_per_image > 0 and len(all_boxes_order[j][i]) >0:
image_scores = np.hstack([all_boxes_order[j][i][:, -1]
for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes_order[j][i][:, -1] >= image_thresh)[0]
all_boxes_order[j][i] = all_boxes_order[j][i][keep, :]
assert (number_im==num_images_detect) # To check that we have the all the images in the detection prediction
det_file = os.path.join(path_data, 'detections.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data +'tmp/' + dataset_nm+'_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
apsAt05 = aps
print("Detection score (thres = 0.5): ",dataset_nm)
print(arrayToLatex(aps,per=True))
ovthresh_tab = [0.3,0.1,0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order, output_dir,ovthresh)
if ovthresh == 0.1:
apsAt01 = aps
print("Detection score with thres at ",ovthresh,'with ',MILmodel)
print(arrayToLatex(aps,per=True))
t2 = time.time()
print("--- Testing duration :",str(t2-t1),' s')
return(apsAt05,apsAt01,AP_per_class)
def run_and_eval_MImax(demonet='res152_COCO',
database='IconArt_v1',
ReDo=True,
PlotRegions=False,
verbose=True,
k_per_bag=300,
CV_Mode=None,
num_split=2,
restarts=11,
max_iters_all_base=300,
LR=0.01,
C=1.0,
Optimizer='GradientDescent',
with_scores=False,
epsilon=0.0,
C_Searching=False,
thresh_evaluation=0.05,
TEST_NMS=0.3,
mini_batch_size=None,
loss_type='',
path_data='data',
path_output='output',
path_to_model='models'):
"""
This function used TFrecords file
Classifier based on CNN features with Transfer Learning on Faster RCNN output
for weakly supervised object detection
Note : with a features maps of 2048, k_bag =300 and a batchsize of 1000 we can
train up to 1200 W vectors in parallel at the same time on a NVIDIA 1080 Ti
@param : demonet : the kind of inside network used it can be 'vgg16_VOC07',
'vgg16_VOC12','vgg16_COCO','res101_VOC12','res101_COCO','res152_COCO'
@param : database : the database used for the weakly supervised detection task
@param : verbose : Verbose option classical
@param : ReDo = False : Erase the former computation, if True and the model already exists
: only do the evaluation
@param : PlotRegions : plot the regions used for learn and the regions in
the positive output response
@param : k_per_bag : number of element per batch in the slection phase [defaut : 300]
@param : CV_Mode : cross validation mode in the MI_max : possibility ;
None, CV in k split
@param : num_split : Number of split for the CV
@param : restarts : number of restarts / reinitialisation in the MI_max [default=11]
@param : max_iters_all_base : number of maximum iteration on the going on
the full database
@param : LR : Learning rate for the optimizer in the MI_max
@param : C : Regularisation term for the optimizer in the MI_max
@param : Optimizer : Optimizer for the MI_max GradientDescent or Adam
@param : thresh_evaluation : 0.05 : seuillage avant de fournir les boites a l evaluation de detections
@param : TEST_NMS : 0.3 : recouvrement autorise avant le NMS avant l evaluation de detections
@param : mini_batch_size if None or 0 an automatic adhoc mini batch size is set
This function output AP for different dataset for the weakly supervised task
"""
item_name, path_to_img, classes, ext, num_classes, str_val, df_label = get_database(
database)
num_trainval_im = len(
df_label[df_label['set'] == 'train'][item_name]) + len(
df_label[df_label['set'] == str_val][item_name])
print('Training on ', database, 'with ', num_trainval_im,
' images in the trainval set')
N = 1
extL2 = ''
nms_thresh = 0.7
savedstr = '_all'
metamodel = 'FasterRCNN'
sets = ['trainval', 'test']
dict_name_file = {}
data_precomputeed = True
if k_per_bag == 300:
k_per_bag_str = ''
else:
k_per_bag_str = '_k' + str(k_per_bag)
for set_str in sets:
name_pkl_all_features = os.path.join(
path_output, metamodel + '_' + demonet + '_' + database + '_N' +
str(N) + extL2 + '_TLforMIL_nms_' + str(nms_thresh) + savedstr +
k_per_bag_str + '_' + set_str + '.tfrecords')
dict_name_file[set_str] = name_pkl_all_features
if set_str in ['trainval', 'test'
] and not (os.path.isfile(name_pkl_all_features)):
data_precomputeed = False
if demonet in ['vgg16_COCO', 'vgg16_VOC07', 'vgg16_VOC12']:
num_features = 4096
elif demonet in ['res101_COCO', 'res152_COCO', 'res101_VOC07', 'res152']:
num_features = 2048
if not (data_precomputeed):
# Compute the features
if verbose:
print(
"We will use a Faster RCNN as feature extractor and region proposals"
)
if metamodel == 'FasterRCNN':
Compute_Faster_RCNN_features(demonet=demonet,
nms_thresh=nms_thresh,
database=database,
augmentation=False,
L2=False,
saved='all',
verbose=verbose,
filesave='tfrecords',
k_regions=k_per_bag)
else:
raise (NotImplementedError)
# Config param for TF session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Data for the MI_max Latent SVM
# All those parameter are design for a GPU 1080 Ti memory size ie 11GB
performance = False
sizeMax = 30 * 200000 // (k_per_bag * 20)
if not (CV_Mode == 'CV' and num_split == 2):
sizeMax //= 2
if num_features > 2048:
sizeMax //= (num_features // 2048)
model_str = 'MI_max'
if k_per_bag == 300:
buffer_size = 10000
else:
buffer_size = 5000 * 300 // k_per_bag
if (k_per_bag > 300 or num_trainval_im > 5000):
usecache = False
else:
usecache = True
if mini_batch_size is None or mini_batch_size == 0:
mini_batch_size = min(sizeMax, num_trainval_im)
max_iters = ((num_trainval_im // mini_batch_size)+ \
np.sign(num_trainval_im % mini_batch_size))*max_iters_all_base
AP_per_class = []
P_per_class = []
R_per_class = []
P20_per_class = []
AP_per_classbS = []
final_clf = None
if C == 1.0:
C_str = ''
else:
C_str = '_C' + str(C) # regularisation term
if with_scores:
with_scores_str = '_WRC' + str(epsilon)
else:
with_scores_str = ''
extPar = '_p'
if CV_Mode == 'CV':
max_iters = (
max_iters * (num_split - 1) // num_split
) # Modification d iteration max par rapport au nombre de split
extCV = '_cv' + str(num_split)
elif CV_Mode is None or CV_Mode == '':
extCV = ''
else:
raise (NotImplementedError)
extCV += '_wr'
if Optimizer == 'Adam':
opti_str = ''
elif Optimizer == 'GradientDescent':
opti_str = '_gd'
elif Optimizer == 'lbfgs':
opti_str = '_lbfgs'
else:
raise (NotImplementedError)
if loss_type is None or loss_type == '':
loss_type_str = ''
elif loss_type == 'hinge':
loss_type_str = 'Losshinge'
if LR == 0.01:
LR_str = ''
else:
LR_str = '_LR' + str(LR)
optimArg = None
if optimArg == None or Optimizer == 'GradientDescent':
optimArg_str = ''
else:
if Optimizer == 'Adam' and str(optimArg).replace(
' ', '_'
) == "{'learning_rate':_0.01,_'beta1':_0.9,_'beta2':_0.999,_'epsilon':_1e-08}":
optimArg_str = ''
else:
optimArg_str = str(optimArg).replace(' ', '_')
verboseMI_max = verbose
shuffle = True
if num_trainval_im == mini_batch_size:
shuffle = False
number_zone = k_per_bag
dont_use_07_metric = True
dim_rois = 5
cachefilefolder = os.path.join(path_data, 'cachefile')
cachefile_model_base='WLS_'+ database+ '_'+demonet+'_r'+str(restarts)+'_s' \
+str(mini_batch_size)+'_k'+str(k_per_bag)+'_m'+str(max_iters)+extPar+\
extCV+opti_str+LR_str+C_str+with_scores_str+ loss_type_str
pathlib.Path(cachefilefolder).mkdir(parents=True, exist_ok=True)
cachefile_model = os.path.join(
cachefilefolder, cachefile_model_base + '_' + model_str + '.pkl')
if verbose: print("cachefile name", cachefile_model)
if not os.path.isfile(cachefile_model) or ReDo:
name_milsvm = {}
if verbose: print("The cachefile doesn t exist or we will erase it.")
else:
with open(cachefile_model, 'rb') as f:
name_milsvm = pickle.load(f)
if verbose: print("The cachefile exists")
usecache_eval = True
if database == 'watercolor':
imdb = get_imdb('watercolor_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database == 'PeopleArt':
imdb = get_imdb('PeopleArt_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database == 'clipart':
imdb = get_imdb('clipart_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database == 'IconArt_v1':
imdb = get_imdb('IconArt_v1_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(df_label[df_label['set'] == 'test'][item_name])
else:
num_images = len(df_label[df_label['set'] == 'test'][item_name])
all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]
data_path_train = dict_name_file['trainval']
if not os.path.isfile(cachefile_model) or ReDo:
if verbose: t0 = time.time()
classifierMI_max = tf_MI_max(LR=LR,
C=C,
restarts=restarts,
num_rois=k_per_bag,
max_iters=max_iters,
buffer_size=buffer_size,
verbose=verboseMI_max,
Optimizer=Optimizer,
mini_batch_size=mini_batch_size,
num_features=num_features,
num_classes=num_classes,
num_split=num_split,
CV_Mode=CV_Mode,
with_scores=with_scores,
epsilon=epsilon,
loss_type=loss_type,
usecache=usecache)
export_dir = classifierMI_max.fit_MI_max_tfrecords(data_path=data_path_train, \
shuffle=shuffle,C_Searching=C_Searching)
if verbose:
t1 = time.time()
print('Total duration training part :', str(t1 - t0))
np_pos_value, np_neg_value = classifierMI_max.get_porportions()
name_milsvm = export_dir, np_pos_value, np_neg_value
with open(cachefile_model, 'wb') as f:
pickle.dump(name_milsvm, f)
else:
export_dir, np_pos_value, np_neg_value = name_milsvm
true_label_all_test,predict_label_all_test,name_all_test,labels_test_predited \
,all_boxes = \
tfR_evaluation_parall(database=database,num_classes=num_classes,
export_dir=export_dir,dict_name_file=dict_name_file,mini_batch_size=mini_batch_size
,config=config,scoreInMI_max=with_scores,
path_to_img=path_to_img,path_data=path_data,classes=classes,
verbose=verbose,thresh_evaluation=thresh_evaluation,TEST_NMS=TEST_NMS,all_boxes=all_boxes
,PlotRegions=PlotRegions,cachefile_model_base=cachefile_model_base,number_im=np.inf,dim_rois=dim_rois,
usecache=usecache_eval,k_per_bag=k_per_bag,num_features=num_features)
for j, classe in enumerate(classes):
AP = average_precision_score(true_label_all_test[:, j],
predict_label_all_test[:, j],
average=None)
print("MI_Max version Average Precision for", classes[j], " = ", AP)
test_precision = precision_score(
true_label_all_test[:, j],
labels_test_predited[:, j],
)
test_recall = recall_score(
true_label_all_test[:, j],
labels_test_predited[:, j],
)
F1 = f1_score(
true_label_all_test[:, j],
labels_test_predited[:, j],
)
print(
"Test on all the data precision = {0:.2f}, recall = {1:.2f},F1 = {2:.2f}"
.format(test_precision, test_recall, F1))
precision_at_k = ranking_precision_score(
np.array(true_label_all_test[:, j]), predict_label_all_test[:, j],
20)
P20_per_class += [precision_at_k]
AP_per_class += [AP]
R_per_class += [test_recall]
P_per_class += [test_precision]
with open(cachefile_model, 'wb') as f:
pickle.dump(name_milsvm, f)
# Detection evaluation
if database in ['watercolor', 'clipart', 'PeopleArt', 'IconArt_v1']:
det_file = os.path.join(path_data, 'cachefile', 'detections_aux.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL)
max_per_image = 100
num_images_detect = len(
imdb.image_index
) # We do not have the same number of images in the WikiTenLabels or IconArt_v1 case
all_boxes_order = [[[] for _ in range(num_images_detect)]
for _ in range(imdb.num_classes)]
number_im = 0
name_all_test = name_all_test.astype(str)
for i in range(num_images_detect):
name_img = imdb.image_path_at(i)
if database == 'PeopleArt':
name_img_wt_ext = name_img.split(
'/')[-2] + '/' + name_img.split('/')[-1]
name_img_wt_ext_tab = name_img_wt_ext.split('.')
name_img_wt_ext = '.'.join(name_img_wt_ext_tab[0:-1])
else:
name_img_wt_ext = name_img.split('/')[-1]
name_img_wt_ext = name_img_wt_ext.split('.')[0]
name_img_ind = np.where(
np.array(name_all_test) == name_img_wt_ext)[0]
#print(name_img_ind)
if len(name_img_ind) == 0:
print('len(name_img_ind), images not found in the all_boxes')
print(name_img_wt_ext)
raise (Exception)
else:
number_im += 1
#print(name_img_ind[0])
for j in range(1, imdb.num_classes):
j_minus_1 = j - 1
all_boxes_order[j][i] = all_boxes[j_minus_1][name_img_ind[0]]
if max_per_image > 0:
image_scores = np.hstack([
all_boxes_order[j][i][:, -1]
for j in range(1, imdb.num_classes)
])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(
all_boxes_order[j][i][:, -1] >= image_thresh)[0]
all_boxes_order[j][i] = all_boxes_order[j][i][keep, :]
assert (
number_im == num_images_detect
) # To check that we have the all the images in the detection prediction
det_file = os.path.join(path_data, 'detections.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data + 'tmp/' + database + '_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
apsAt05 = aps
print("Detection score (thres = 0.5): ", database,
'with MI_Max with score =', with_scores)
print(arrayToLatex(aps, per=True))
ovthresh_tab = [0.3, 0.1, 0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order,
output_dir, ovthresh)
if ovthresh == 0.1:
apsAt01 = aps
print("Detection score with thres at ", ovthresh,
'with MI_Max with score =', with_scores)
print(arrayToLatex(aps, per=True))
print('~~~~~~~~')
print("mean Average Precision Classification for all the data = {0:.3f}".
format(np.mean(AP_per_class)))
print("mean Precision Classification for all the data = {0:.3f}".format(
np.mean(P_per_class)))
print("mean Recall Classification for all the data = {0:.3f}".format(
np.mean(R_per_class)))
#print("mean Precision Classification @ 20 for all the data = {0:.3f}".format(np.mean(P20_per_class)))
print('Mean Average Precision Classification with MI_Max with score =',
with_scores, ' : ')
print(AP_per_class)
print(arrayToLatex(AP_per_class, per=True))
return (apsAt05, apsAt01, AP_per_class)
def LearnOn30andTestOn300():
""" This function is just used to do the test with a model learn on 30 regions"""
demonet = 'res152_COCO'
database = 'WikiTenLabels'
verbose = True
testMode = False
jtest = 'cow'
PlotRegions = False
saved_clf=False
RPN=False
CompBest=False
Stocha=True
k_per_bag=30
parallel_op=True
CV_Mode=''
num_split=2
WR=True
init_by_mean =None
seuil_estimation=''
restarts=11
max_iters_all_base=300
LR=0.01
with_tanh=True
C=1.0
Optimizer='GradientDescent'
norm=''
transform_output='tanh'
with_rois_scores_atEnd=False
optim_wt_Reg = False
with_scores=False
epsilon=0.01
restarts_paral='paral'
Max_version=''
w_exp=10.0
seuillage_by_score=False
seuil=0.9
k_intopk=1
C_Searching=False
predict_with='MILSVM'
gridSearch=False
thres_FinalClassifier=0.5
n_jobs=1
thresh_evaluation=0.05
TEST_NMS=0.3
ext = '.txt'
if database=='Paintings':
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/Painting_Dataset/'
classes = ['aeroplane','bird','boat','chair','cow','diningtable','dog','horse','sheep','train']
elif database=='VOC12':
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/VOCdevkit/VOC2012/JPEGImages/'
elif database=='VOC2007':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/VOCdevkit/VOC2007/JPEGImages/'
classes = ['aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat', 'chair',
'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor']
elif database=='watercolor':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/cross-domain-detection/datasets/watercolor/JPEGImages/'
classes = ["bicycle", "bird","car", "cat", "dog", "person"]
elif database=='PeopleArt':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/PeopleArt/JPEGImages/'
classes = ["person"]
elif database=='WikiTenLabels':
ext = '.csv'
item_name = 'item'
path_to_img = '/media/gonthier/HDD/data/Wikidata_Paintings/WikiTenLabels/JPEGImages/'
classes = ['angel', 'beard','capital','Child_Jesus', 'crucifixion_of_Jesus',
'Mary','nudity', 'ruins','Saint_Sebastien','turban']
elif database=='clipart':
ext = '.csv'
item_name = 'name_img'
path_to_img = '/media/gonthier/HDD/data/cross-domain-detection/datasets/clipart/JPEGImages/'
classes = ['aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat', 'chair',
'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor']
elif(database=='Wikidata_Paintings'):
item_name = 'image'
path_to_img = '/media/gonthier/HDD/data/Wikidata_Paintings/600/'
raise NotImplemented # TODO implementer cela !!!
elif(database=='Wikidata_Paintings_miniset_verif'):
item_name = 'image'
path_to_img = '/media/gonthier/HDD/data/Wikidata_Paintings/600/'
classes = ['Q235113_verif','Q345_verif','Q10791_verif','Q109607_verif','Q942467_verif']
if testMode and not(type(jtest)==int):
assert(type(jtest)==str)
jtest = int(np.where(np.array(classes)==jtest)[0][0])# Conversion of the jtest string to the value number
assert(type(jtest)==int)
if testMode and (jtest>len(classes)) :
print("We are in test mode but jtest>len(classes), we will use jtest =0" )
jtest =0
path_data = '/media/gonthier/HDD/output_exp/ClassifPaintings/'
databasetxt =path_data + database + ext
if database=='WikiTenLabels':
dtypes = {0:str,'item':str,'angel':int,'beard':int,'capital':int, \
'Child_Jesus':int,'crucifixion_of_Jesus':int,'Mary':int,'nudity':int,'ruins':int,'Saint_Sebastien':int,\
'turban':int,'set':str,'Anno':int}
df_label = pd.read_csv(databasetxt,sep=",",dtype=dtypes)
else:
df_label = pd.read_csv(databasetxt,sep=",")
str_val = 'val'
if database=='Wikidata_Paintings_miniset_verif':
df_label = df_label[df_label['BadPhoto'] <= 0.0]
str_val = 'validation'
elif database=='Paintings':
str_val = 'validation'
elif database in ['VOC2007','watercolor','clipart','PeopleArt']:
str_val = 'val'
df_label[classes] = df_label[classes].apply(lambda x:(x + 1.0)/2.0)
num_trainval_im = len(df_label[df_label['set']=='train'][item_name]) + len(df_label[df_label['set']==str_val][item_name])
num_classes = len(classes)
print(database,'with ',num_trainval_im,' images in the trainval set')
N = 1
extL2 = ''
nms_thresh = 0.7
savedstr = '_all'
thresh_evaluation = 0.05
TEST_NMS =0.3
thresh = 0.05
sets = ['train','val','trainval','test']
dict_name_file = {}
data_precomputeed= True
if k_per_bag==300:
k_per_bag_str = ''
else:
k_per_bag_str = '_k'+str(k_per_bag)
eval_onk300 = True
for set_str in sets:
name_pkl_all_features = path_data+'FasterRCNN_'+ demonet +'_'+database+'_N'+str(N)+extL2+'_TLforMIL_nms_'+str(nms_thresh)+savedstr+k_per_bag_str+'_'+set_str+'.tfrecords'
if not(k_per_bag==300) and eval_onk300 and set_str=='test': # We will evaluate on all the 300 regions and not only the k_per_bag ones
name_pkl_all_features = path_data+'FasterRCNN_'+ demonet +'_'+database+'_N'+str(N)+extL2+'_TLforMIL_nms_'+str(nms_thresh)+savedstr+'_'+set_str+'.tfrecords'
dict_name_file[set_str] = name_pkl_all_features
if not(os.path.isfile(name_pkl_all_features)):
data_precomputeed = False
# sLength_all = len(df_label[item_name])
if demonet == 'vgg16_COCO':
num_features = 4096
elif demonet in ['res101_COCO','res152_COCO','res101_VOC07']:
num_features = 2048
if not(data_precomputeed):
# Compute the features
if verbose: print("We will computer the CNN features")
Compute_Faster_RCNN_features(demonet=demonet,nms_thresh =nms_thresh,
database=database,augmentation=False,L2 =False,
saved='all',verbose=verbose,filesave='tfrecords',k_regions=k_per_bag)
dont_use_07_metric = True
if database=='VOC2007':
imdb = get_imdb('voc_2007_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='watercolor':
imdb = get_imdb('watercolor_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='PeopleArt':
imdb = get_imdb('PeopleArt_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='clipart':
imdb = get_imdb('clipart_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
num_images = len(imdb.image_index)
elif database=='WikiTenLabels':
imdb = get_imdb('WikiTenLabels_test')
imdb.set_force_dont_use_07_metric(dont_use_07_metric)
#num_images = len(imdb.image_index)
num_images = len(df_label[df_label['set']=='test'][item_name])
else:
num_images = len(df_label[df_label['set']=='test'][item_name])
all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)]
data_path_train= dict_name_file['trainval']
# Config param for TF session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Data for the MILSVM Latent SVM
# All those parameter are design for my GPU 1080 Ti memory size
performance = False
if parallel_op:
sizeMax = 30*10000 // (k_per_bag*num_classes)
else:
sizeMax = 30*10000 // k_per_bag
if not(init_by_mean is None) and not(init_by_mean==''):
if not(CV_Mode=='CV' and num_split==2):
sizeMax //= 2
# boolean paralleliation du W
if not(num_features==2048):
sizeMax //= (num_features//2048)
if restarts_paral=='Dim': # It will create a new dimension
restarts_paral_str = '_RP'
sizeMax //= max(int((restarts+1)//2),1) # To avoid division by zero
# it seems that using a different size batch drasticly change the results
elif restarts_paral=='paral': # Version 2 of the parallelisation
restarts_paral_str = '_RPV2'
sizeMax = 30*200000 // (k_per_bag*20)
else:
restarts_paral_str=''
# InternalError: Dst tensor is not initialized. can mean that you are running out of GPU memory
mini_batch_size = min(sizeMax,num_trainval_im)
buffer_size = 10000
if testMode:
ext_test = '_Test_Mode'
else:
ext_test= ''
max_iters = ((num_trainval_im // mini_batch_size)+ \
np.sign(num_trainval_im % mini_batch_size))*max_iters_all_base
AP_per_class = []
P_per_class = []
R_per_class = []
P20_per_class = []
AP_per_classbS = []
final_clf = None
class_weight = None
ReDo = False
if C_Searching:C_Searching_str ='_Csearch'
else: C_Searching_str = ''
if with_scores:
with_scores_str = '_WRC'+str(epsilon)
else:
with_scores_str=''
if seuillage_by_score:
seuillage_by_score_str = '_SBS'+str(seuil)
else: seuillage_by_score_str = ''
if norm=='L2':
extNorm = '_L2'
elif norm=='STDall':
extNorm = '_STDall'
elif norm=='STDSaid':
extNorm = '_STDSaid'
elif norm=='STD':
extNorm = '_STD'
raise(NotImplemented)
elif norm=='' or norm is None:
extNorm = ''
if parallel_op:
extPar = '_p'
else:
extPar =''
if CV_Mode=='CV':
max_iters = (max_iters*(num_split-1)//num_split) # Modification d iteration max par rapport au nombre de split
extCV = '_cv'+str(num_split)
elif CV_Mode=='LA':
max_iters = (max_iters*(num_split-1)//num_split) # Modification d iteration max par rapport au nombre de split
extCV = '_la'+str(num_split)
elif CV_Mode=='CV' and WR==True:
extCV = '_cv'+str(num_split)+'_wr'
elif CV_Mode is None or CV_Mode=='':
extCV =''
if WR: extCV += '_wr'
if Optimizer=='Adam':
opti_str=''
elif Optimizer=='GradientDescent':
opti_str='_gd'
else:
raise(NotImplemented)
if init_by_mean is None or init_by_mean=='':
init_by_mean_str = ''
elif init_by_mean=='First':
init_by_mean_str= '_ibnF'
elif init_by_mean=='All':
init_by_mean_str= '_ibnA'
if LR==0.01:
LR_str = ''
else:
LR_str='_LR'+str(LR)
if C == 1.0:
C_str=''
else:
C_str = '_C'+str(C) # regularisation term
if Max_version=='max' or Max_version=='' or Max_version is None:
Max_version_str =''
elif Max_version=='softmax':
Max_version_str ='_MVSF'
if not(w_exp==1.0): Max_version_str+=str(w_exp)
elif Max_version=='sparsemax':
Max_version_str ='_MVSM'
elif Max_version=='mintopk':
Max_version_str ='_MVMT'+str(k_intopk)
optimArg= None
verboseMILSVM = True
shuffle = True
if num_trainval_im==mini_batch_size:
shuffle = False
if shuffle:
shuffle_str = ''
else:
shuffle_str = '_allBase'
if optim_wt_Reg:
optim_wt_Reg_str = '_OptimWTReg'
else:
optim_wt_Reg_str =''
Number_of_positif_elt = 1
number_zone = k_per_bag
# thresh_evaluation,TEST_NMS = 0.05,0.3
dont_use_07_metric = True
symway = True
arrayParam = [demonet,database,N,extL2,nms_thresh,savedstr,mini_batch_size,
performance,buffer_size,predict_with,shuffle,C,testMode,restarts,max_iters_all_base,
max_iters,CV_Mode,num_split,parallel_op,WR,norm,Optimizer,LR,optimArg,
Number_of_positif_elt,number_zone,seuil_estimation,thresh_evaluation,
TEST_NMS,init_by_mean,transform_output,with_rois_scores_atEnd,
with_scores,epsilon,restarts_paral,Max_version,w_exp,seuillage_by_score,seuil,
k_intopk,C_Searching,gridSearch,thres_FinalClassifier,optim_wt_Reg]
arrayParamStr = ['demonet','database','N','extL2','nms_thresh','savedstr',
'mini_batch_size','performance','buffer_size','predict_with',
'shuffle','C','testMode','restarts','max_iters_all_base','max_iters','CV_Mode',
'num_split','parallel_op','WR','norm','Optimizer','LR',
'optimArg','Number_of_positif_elt','number_zone','seuil_estimation'
,'thresh_evaluation','TEST_NMS','init_by_mean','transform_output','with_rois_scores_atEnd',
'with_scores','epsilon','restarts_paral','Max_version','w_exp','seuillage_by_score',
'seuil','k_intopk','C_Searching','gridSearch','thres_FinalClassifier','optim_wt_Reg']
assert(len(arrayParam)==len(arrayParamStr))
print(tabs_to_str(arrayParam,arrayParamStr))
# print('database',database,'mini_batch_size',mini_batch_size,'max_iters',max_iters,'norm',norm,\
# 'parallel_op',parallel_op,'CV_Mode',CV_Mode,'WR',WR,'restarts',restarts,'demonet',demonet,
# 'Optimizer',Optimizer,'init_by_mean',init_by_mean,'with_tanh',with_tanh)
cachefile_model_base= database +'_'+demonet+'_r'+str(restarts)+'_s' \
+str(mini_batch_size)+'_k'+str(k_per_bag)+'_m'+str(max_iters)+extNorm+extPar+\
extCV+ext_test+opti_str+LR_str+C_str+init_by_mean_str+with_scores_str+restarts_paral_str\
+Max_version_str+seuillage_by_score_str+shuffle_str+C_Searching_str+optim_wt_Reg_str
cachefile_model = path_data + cachefile_model_base+'_MILSVM.pkl'
with open(cachefile_model, 'rb') as f:
name_milsvm = pickle.load(f)
export_dir,np_pos_value,np_neg_value= name_milsvm
export_dir_path = ('/').join(export_dir.split('/')[:-1])
name_model_meta = export_dir + '.meta'
get_roisScore = False
train_dataset = tf.data.TFRecordDataset(dict_name_file['test'])
if not(k_per_bag==300) and eval_onk300:
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore,num_features=num_features,num_rois=300))
else:
train_dataset = train_dataset.map(lambda r: parser_w_rois_all_class(r,\
num_classes=num_classes,with_rois_scores=get_roisScore,num_features=num_features,num_rois=k_per_bag))
dataset_batch = train_dataset.batch(mini_batch_size)
dataset_batch.cache()
iterator = dataset_batch.make_one_shot_iterator()
next_element = iterator.get_next()
true_label_all_test = []
predict_label_all_test = []
name_all_test = []
FirstTime= True
i = 0
ii = 0
load_model = False
scoreInMILSVM =False
seuil_estimation_bool = False
with_rois_scores_atEnd=False
seuillage_by_score=False
gridSearch=False
with_softmax_a_intraining= False
with_softmax = False
sess = tf.Session(config=config)
if load_model==False:
new_saver = tf.train.import_meta_graph(name_model_meta)
new_saver.restore(sess, tf.train.latest_checkpoint(export_dir_path))
graph= tf.get_default_graph()
X = graph.get_tensor_by_name("X:0")
y = graph.get_tensor_by_name("y:0")
if scoreInMILSVM:
scores_tf = graph.get_tensor_by_name("scores:0")
Prod_best = graph.get_tensor_by_name("ProdScore:0")
else:
Prod_best = graph.get_tensor_by_name("Prod:0")
if with_tanh:
print('use of tanh')
Tanh = tf.tanh(Prod_best)
mei = tf.argmax(Tanh,axis=2)
score_mei = tf.reduce_max(Tanh,axis=2)
elif with_softmax:
Softmax = tf.nn.softmax(Prod_best,axis=-1)
mei = tf.argmax(Softmax,axis=2)
score_mei = tf.reduce_max(Softmax,axis=2)
elif with_softmax_a_intraining:
Softmax=tf.multiply(tf.nn.softmax(Prod_best,axis=-1),Prod_best)
mei = tf.argmax(Softmax,axis=2)
score_mei = tf.reduce_max(Softmax,axis=2)
else:
mei = tf.argmax(Prod_best,axis=2)
score_mei = tf.reduce_max(Prod_best,axis=2)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
while True:
try:
if not(with_rois_scores_atEnd) and not(scoreInMILSVM):
fc7s,roiss, labels,name_imgs = sess.run(next_element)
else:
fc7s,roiss,rois_scores,labels,name_imgs = sess.run(next_element)
fc7s_full,roiss_full, labels_full,name_imgs_full = fc7s,roiss, labels,name_imgs
PositiveExScoreAll_tab = []
for iii in range(10):
fc7s = fc7s_full[:,iii*30:iii*30+30,:]
roiss = roiss_full[:,iii*30:iii*30+30]
labels = labels_full
name_imgs = name_imgs_full
if predict_with=='MILSVM':
if scoreInMILSVM:
feed_dict_value = {X: fc7s,scores_tf: rois_scores, y: labels}
else:
feed_dict_value = {X: fc7s, y: labels}
if with_tanh:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Tanh], feed_dict=feed_dict_value)
elif with_softmax or with_softmax_a_intraining:
PositiveRegions,get_RegionsScore,PositiveExScoreAll =\
sess.run([mei,score_mei,Softmax], feed_dict=feed_dict_value)
else:
PositiveRegions,get_RegionsScore,PositiveExScoreAll = \
sess.run([mei,score_mei,Prod_best], feed_dict=feed_dict_value)
if with_rois_scores_atEnd:
PositiveExScoreAll = PositiveExScoreAll*rois_scores
get_RegionsScore = np.max(PositiveExScoreAll,axis=2)
PositiveRegions = np.amax(PositiveExScoreAll,axis=2)
if with_tanh: assert(np.max(PositiveExScoreAll) <= 1.)
PositiveExScoreAll_tab += [PositiveExScoreAll]
roiss = roiss_full
PositiveExScoreAll= np.concatenate(PositiveExScoreAll_tab,axis=2)
PositiveRegions = np.amax(PositiveExScoreAll,axis=2)
get_RegionsScore= np.max(PositiveExScoreAll,axis=2)
print('PositiveExScoreAll',PositiveExScoreAll.shape)
print('PositiveRegions',PositiveRegions.shape)
print('get_RegionsScore',get_RegionsScore.shape)
print('name_imgs',len(name_imgs))
true_label_all_test += [labels]
if predict_with=='MILSVM':
predict_label_all_test += [get_RegionsScore]
elif 'LinearSVC' in predict_with:
predict_label_all_test_tmp = []
for j in range(num_classes):
predict_label_all_test_tmp += [np.reshape(classifier_trained_dict[j].decision_function(
np.reshape(fc7s,(-1,fc7s.shape[-1]))),(fc7s.shape[0],fc7s.shape[1]))]
predict_label_all_test_batch = np.stack(predict_label_all_test_tmp,axis=0)
# print(predict_label_all_test_batch.shape)
predict_label_all_test += [np.max(predict_label_all_test_batch,axis=2)]
# print('predict_label_all_test',predict_label_all_test[-1].shape)
# predict_label_all_test is used only for the classification score !
# if predict_with=='LinearSVC':
for k in range(len(labels)):
if database in ['VOC2007','watercolor','Paintings','clipart','WikiTenLabels','PeopleArt'] :
complet_name = path_to_img + str(name_imgs[k].decode("utf-8")) + '.jpg'
else:
complet_name = path_to_img + name_imgs[k] + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
if predict_with=='MILSVM':
scores_all = PositiveExScoreAll[:,k,:]
elif 'LinearSVC' in predict_with:
scores_all = predict_label_all_test_batch[:,k,:]
roi = roiss[k,:]
roi_boxes = roi[:,1:5] / im_scales[0]
for j in range(num_classes):
scores = scores_all[j,:]
if seuil_estimation_bool:
inds = np.where(scores > list_thresh[j])[0]
else:
inds = np.where(scores > thresh)[0]
cls_scores = scores[inds]
cls_boxes = roi_boxes[inds,:]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32, copy=False)
modif_box = '' # Possibility : SumPond, Inter
if(not(modif_box=='') and not(modif_box is None)):
# Modification of the bounding box
cls_dets = py_cpu_modif(cls_dets,kind=modif_box)
keep = nms(cls_dets, TEST_NMS)
cls_dets = cls_dets[keep, :]
all_boxes[j][i] = cls_dets
i+=1
for l in range(len(name_imgs)):
if database in ['VOC2007','watercolor','clipart','WikiTenLabels','PeopleArt']:
name_all_test += [[str(name_imgs[l].decode("utf-8"))]]
else:
name_all_test += [[name_imgs[l]]]
if PlotRegions and predict_with=='MILSVM':
if verbose and (ii%1000==0):
print("Plot the images :",ii)
if verbose and FirstTime:
FirstTime = False
print("Start ploting Regions on test set")
for k in range(len(labels)):
if ii > number_im:
continue
if database in ['VOC2007','Paintings','watercolor','clipart','WikiTenLabels','PeopleArt']:
name_img = str(name_imgs[k].decode("utf-8") )
else:
name_img = name_imgs[k]
rois = roiss[k,:]
if database in ['VOC12','Paintings','VOC2007','clipart','watercolor','WikiTenLabels','PeopleArt']:
complet_name = path_to_img + name_img + '.jpg'
name_sans_ext = name_img
elif(database=='Wikidata_Paintings') or (database=='Wikidata_Paintings_miniset_verif'):
name_sans_ext = os.path.splitext(name_img)[0]
complet_name = path_to_img +name_sans_ext + '.jpg'
im = cv2.imread(complet_name)
blobs, im_scales = get_blobs(im)
roi_boxes_and_score = []
local_cls = []
for j in range(num_classes):
cls_dets = all_boxes[j][ii] # Here we have #classe x box dim + score
# Last element is the score
# print(cls_dets.shape)
if len(cls_dets) > 0:
local_cls += [classes[j]]
# roi_boxes_score = np.expand_dims(cls_dets,axis=0)
roi_boxes_score = cls_dets
# print(roi_boxes_score.shape)
if roi_boxes_and_score is None:
roi_boxes_and_score = [roi_boxes_score]
else:
roi_boxes_and_score += [roi_boxes_score]
#np.vstack((roi_boxes_and_score,roi_boxes_score))
if roi_boxes_and_score is None: roi_boxes_and_score = [[]]
ii += 1
if RPN:
best_RPN_roi = rois[0,:]
best_RPN_roi_boxes = best_RPN_roi[1:5] / im_scales[0]
best_RPN_roi_scores = [PositiveExScoreAll[j,k,0]]
cls = local_cls + ['RPN'] # Comparison of the best region according to the faster RCNN and according to the MILSVM de Said
#best_RPN_roi_boxes_score = np.expand_dims(np.expand_dims(np.concatenate((best_RPN_roi_boxes,best_RPN_roi_scores)),axis=0),axis=0)
best_RPN_roi_boxes_score = np.expand_dims(np.concatenate((best_RPN_roi_boxes,best_RPN_roi_scores)),axis=0)
# print(best_RPN_roi_boxes_score.shape)
#roi_boxes_and_score = np.vstack((roi_boxes_and_score,best_RPN_roi_boxes_score))
roi_boxes_and_score += [best_RPN_roi_boxes_score] #np.vstack((roi_boxes_and_score,best_RPN_roi_boxes_score))
else:
cls = local_cls
#print(len(cls),len(roi_boxes_and_score))
vis_detections_list(im, cls, roi_boxes_and_score, thresh=-np.inf)
name_output = path_to_output2 +'Test/' + name_sans_ext + '_Regions.jpg'
if database=='PeopleArt':
path_tmp = '/'.join(name_output.split('/')[0:-1])
pathlib.Path(path_tmp).mkdir(parents=True, exist_ok=True)
plt.savefig(name_output)
plt.close()
except tf.errors.OutOfRangeError:
break
print('End compute')
tf.reset_default_graph()
true_label_all_test = np.concatenate(true_label_all_test)
predict_label_all_test = np.transpose(np.concatenate(predict_label_all_test,axis=1))
name_all_test = np.concatenate(name_all_test)
labels_test_predited = (np.sign(predict_label_all_test) +1.)/2
labels_test_predited[np.where(labels_test_predited==0.5)] = 0 # To deal with the case where predict_label_all_test == 0
for j,classe in enumerate(classes):
AP = average_precision_score(true_label_all_test[:,j],predict_label_all_test[:,j],average=None)
print("MIL-SVM version Average Precision for",classes[j]," = ",AP)
test_precision = precision_score(true_label_all_test[:,j],labels_test_predited[:,j],)
test_recall = recall_score(true_label_all_test[:,j],labels_test_predited[:,j],)
F1 = f1_score(true_label_all_test[:,j],labels_test_predited[:,j],)
print("Test on all the data precision = {0:.2f}, recall = {1:.2f},F1 = {2:.2f}".format(test_precision,test_recall,F1))
precision_at_k = ranking_precision_score(np.array(true_label_all_test[:,j]), predict_label_all_test[:,j],20)
P20_per_class += [precision_at_k]
AP_per_class += [AP]
R_per_class += [test_recall]
P_per_class += [test_precision]
det_file = os.path.join(path_data, 'detections_aux.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL)
max_per_image = 100
num_images_detect = len(imdb.image_index) # We do not have the same number of images in the WikiTenLabels case
all_boxes_order = [[[] for _ in range(num_images_detect)] for _ in range(imdb.num_classes)]
number_im = 0
for i in range(num_images_detect):
name_img = imdb.image_path_at(i)
if database=='PeopleArt':
name_img_wt_ext = name_img.split('/')[-2] +'/' +name_img.split('/')[-1]
name_img_wt_ext_tab =name_img_wt_ext.split('.')
name_img_wt_ext = '.'.join(name_img_wt_ext_tab[0:-1])
else:
name_img_wt_ext = name_img.split('/')[-1]
name_img_wt_ext =name_img_wt_ext.split('.')[0]
name_img_ind = np.where(np.array(name_all_test)==name_img_wt_ext)[0]
#print(name_img_ind)
if len(name_img_ind)==0:
print('len(name_img_ind), images not found in the all_boxes')
print(name_img_wt_ext)
raise(Exception)
else:
number_im += 1
#print(name_img_ind[0])
for j in range(1, imdb.num_classes):
j_minus_1 = j-1
all_boxes_order[j][i] = all_boxes[j_minus_1][name_img_ind[0]]
if max_per_image > 0:
image_scores = np.hstack([all_boxes_order[j][i][:, -1]
for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes_order[j][i][:, -1] >= image_thresh)[0]
all_boxes_order[j][i] = all_boxes_order[j][i][keep, :]
assert (number_im==num_images_detect) # To check that we have the all the images in the detection prediction
det_file = os.path.join(path_data, 'detections.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes_order, f, pickle.HIGHEST_PROTOCOL)
output_dir = path_data +'tmp/' + database+'_mAP.txt'
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
print("Detection score (thres = 0.5): ",database)
print(arrayToLatex(aps,per=True))
ovthresh_tab = [0.3,0.1,0.]
for ovthresh in ovthresh_tab:
aps = imdb.evaluate_localisation_ovthresh(all_boxes_order, output_dir,ovthresh)
print("Detection score with thres at ",ovthresh)
print(arrayToLatex(aps,per=True))
imdb.set_use_diff(True) # Modification of the use_diff attribute in the imdb
aps = imdb.evaluate_detections(all_boxes_order, output_dir)
print("Detection score with the difficult element")
print(arrayToLatex(aps,per=True))
imdb.set_use_diff(False)
print('~~~~~~~~')
print("mean Average Precision Classification for all the data = {0:.3f}".format(np.mean(AP_per_class)))
if CompBest: print("mean Average Precision for BEst Score = {0:.3f}".format(np.mean(AP_per_classbS)))
print("mean Precision Classification for all the data = {0:.3f}".format(np.mean(P_per_class)))
print("mean Recall Classification for all the data = {0:.3f}".format(np.mean(R_per_class)))
print("mean Precision Classification @ 20 for all the data = {0:.3f}".format(np.mean(P20_per_class)))
print('Mean Average Precision Classification :')
print(AP_per_class)
print(arrayToLatex(AP_per_class,per=True))
return(0)