def script_visualizePerformanceDifference(): trained_file = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/trained/20151027204114' notrained_file = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/no_trained/20151027203547' out_file = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/nn_performance_comparison_trained_notrained.png' out_file_diff = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/nn_performance_comparison_trained_notrained_diff.png' file_pres = [trained_file, notrained_file] layers = ['pool5', 'fc6', 'fc7'] legend_pres = ['Trained', 'Unfamiliar'] legend_entries = [] for leg in legend_pres: legend_entries.extend([leg + ' ' + layer for layer in layers]) vecs_to_plot = [] file_names = [ file_pre + '_' + layer for file_pre in file_pres for layer in layers ] for idx, file_name in enumerate(file_names): print file_name, legend_entries[idx] [img_paths, gt_labels, indices, _] = pickle.load(open(file_name + '.p', 'rb')) no_correct, _ = script_nearestNeigbourExperiment.getNumberOfCorrectNNMatches( indices, gt_labels) with open(file_name + '.txt', 'wb') as f: for num in no_correct: f.write(str(num) + ' ') f.write('\n') # with open(file_name+'.txt','rb') as f: # no_correct=f.readline(); # no_correct=[float(no_correct_curr) for no_correct_curr in no_correct.strip('\n').split()]; vecs_to_plot.append(no_correct) # print legend_entries; print vecs_to_plot # print len(vecs_to_plot); # print len(legend_entries); plt.figure() plt.xlabel('Number of Nearest Neighbours K') plt.ylabel('Accuracy') plt.title('NN Accuracy DNN Features for Pascal Classes') plt.xlim(0, 6) plt.ylim( min([min(vec) for vec in vecs_to_plot]) - 0.05, max([max(vec) for vec in vecs_to_plot]) + 0.05) handles = [] for vec in vecs_to_plot: handle, = plt.plot(range(1, len(vec) + 1), vec) handles.append(handle) plt.legend(handles, legend_entries, loc=2, prop={'size': 10}) plt.savefig(out_file) legend_entries = ['Trained-Untrained ' + layer for layer in layers] diffs = [] for idx in range(3): a = vecs_to_plot[idx] b = vecs_to_plot[idx + 3] diff = [a[idx_curr] - b[idx_curr] for idx_curr in range(len(a))] diffs.append(diff) vecs_to_plot = diffs plt.figure() plt.xlabel('Number of Nearest Neighbours K') plt.ylabel('Accuracy Difference') plt.title('NN Accuracy DNN Features for Pascal Classes') plt.xlim(0, 6) plt.ylim( min([min(vec) for vec in vecs_to_plot]) - 0.01, max([max(vec) for vec in vecs_to_plot]) + 0.01) handles = [] for vec in vecs_to_plot: handle, = plt.plot(range(1, len(vec) + 1), vec) handles.append(handle) plt.legend(handles, legend_entries, loc=2, prop={'size': 10}) plt.savefig(out_file_diff)
def script_runNNOnPascalExcludedInTraining(): path_to_file = '../../data/ilsvrc12/synset_words.txt' val_ids = imagenet.readLabelsFile(path_to_file) val_just_ids = list(zip(*val_ids)[0]) val_just_labels = list(zip(*val_ids)[1]) pascal_ids_file = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_classes.txt' pascal_ids = imagenet.readLabelsFile(pascal_ids_file) pascal_just_ids = list(zip(*pascal_ids)[0]) to_exclude = imagenet.removeClassesWithOverlap(val_just_ids, pascal_just_ids, keepMapping=True) val_gt_file = '../../data/ilsvrc12/val.txt' list_of_ids_im = [id for id_list in to_exclude for id in id_list] mapping_file = '../../data/ilsvrc12/synsets.txt' print len(list_of_ids_im) list_of_ids, _ = imagenet.getImagenetIdToTrainingIdMapping( mapping_file, list_of_ids_im) print len(list_of_ids) # print list_of_ids[0] list_of_ids_pascal = [] for id_no in range(len(to_exclude)): list_of_ids_pascal = list_of_ids_pascal + [id_no] * len( to_exclude[id_no]) path_to_val = '/disk2/imagenet/val' test_set = imagenet.selectTestSetByID(val_gt_file, list_of_ids, path_to_val) # out_dir='/disk2/octoberExperiments/nn_performance_without_pascal/notrained' out_dir = '/disk2/novemberExperiments/nn_imagenet_top5/trained' if not os.path.exists(out_dir): os.mkdir(out_dir) layers = ['pool5', 'fc6', 'fc7'] gpu_no = 1 path_to_classify = '..' numberOfN = 5 relativePaths = ['/disk2', '../../../..'] # deployFile='/disk2/octoberExperiments/nn_performance_without_pascal/deploy.prototxt' # meanFile='/disk2/octoberExperiments/nn_performance_without_pascal/mean.npy' # modelFile='/disk2/octoberExperiments/nn_performance_without_pascal/snapshot_iter_450000.caffemodel' modelFile = '/home/maheenrashid/Downloads/caffe/caffe-rc2/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel' deployFile = '/home/maheenrashid/Downloads/caffe/caffe-rc2/models/bvlc_reference_caffenet/deploy.prototxt' meanFile = '/home/maheenrashid/Downloads/caffe/caffe-rc2/python/caffe/imagenet/ilsvrc_2012_mean.npy' # modelFile='/disk2/novemberExperiments/network_no_pascal/snapshots/snapshot_iter_450000.caffemodel'; # deployFile='/disk2/novemberExperiments/network_no_pascal/deploy.prototxt'; # meanFile='/disk2/novemberExperiments/network_no_pascal/mean.npy'; # out_file=script_nearestNeigbourExperiment.runClassificationTestSet(test_set,out_dir,path_to_classify,gpu_no,layers,deployFile=deployFile,meanFile=meanFile,modelFile=modelFile) # return # file_name='/disk2/octoberExperiments/nn_performance_without_pascal/notrained/20151026132705' # file_name='/disk2/novemberExperiments/nn_imagenet_top5/notrained/20151130193757'; file_name = '/disk2/novemberExperiments/nn_imagenet_top5/trained/20151130230243' file_text_labels = '../../data/ilsvrc12/synset_words.txt' text_labels = np.loadtxt(file_text_labels, str, delimiter='\t') vals = np.load(file_name + '.npz') test_set = sorted(test_set, key=lambda x: x[0]) test_set = zip(*test_set) img_paths = list(test_set[0]) gt_labels = list(test_set[1]) gt_labels_pascal = [ list_of_ids_pascal[list_of_ids.index(gt_label)] for gt_label in gt_labels ] for layer in layers: print layer file_name_l = file_name + '_' + layer indices = script_nearestNeigbourExperiment.doNN(img_paths, gt_labels, vals[layer], numberOfN=numberOfN, distance='cosine', algo='brute') conf_matrix = 0 pickle.dump([img_paths, gt_labels, indices, conf_matrix], open(file_name_l + '.p', 'wb')) file_text_labels_pascal = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_classes.txt' text_labels_pascal = np.loadtxt(file_text_labels_pascal, str, delimiter='\t') for layer in layers: print layer file_name_l = file_name + '_' + layer [img_paths, gt_labels, indices, _] = pickle.load(open(file_name_l + '.p', 'rb')) img_paths_curr = [ x.replace(relativePaths[0], relativePaths[1]) for x in img_paths ] im_paths, captions = script_nearestNeigbourExperiment.createImageAndCaptionGrid( img_paths_curr, gt_labels, indices, text_labels) script_nearestNeigbourExperiment.writeHTML(file_name_l + '.html', im_paths, captions) no_correct, _ = script_nearestNeigbourExperiment.getNumberOfCorrectNNMatches( indices, gt_labels) print no_correct with open(file_name_l + '.txt', 'wb') as f: for no_correct_curr in no_correct: f.write(str(no_correct_curr) + ' ') file_name_l = file_name + '_' + layer + '_pascal' im_paths, captions = script_nearestNeigbourExperiment.createImageAndCaptionGrid( img_paths_curr, gt_labels_pascal, indices, text_labels_pascal) script_nearestNeigbourExperiment.writeHTML(file_name_l + '.html', im_paths, captions) no_correct, _ = script_nearestNeigbourExperiment.getNumberOfCorrectNNMatches( indices, gt_labels_pascal) with open(file_name_l + '.txt', 'wb') as f: for no_correct_curr in no_correct: f.write(str(no_correct_curr) + ' ') print no_correct
def script_visualizePerformanceDifference(): trained_file='/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/trained/20151027204114' notrained_file='/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/no_trained/20151027203547' out_file='/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/nn_performance_comparison_trained_notrained.png'; out_file_diff='/disk2/octoberExperiments/nn_performance_without_pascal/pascal_3d/nn_performance_comparison_trained_notrained_diff.png'; file_pres=[trained_file,notrained_file] layers=['pool5','fc6','fc7']; legend_pres=['Trained','Unfamiliar']; legend_entries=[]; for leg in legend_pres: legend_entries.extend([leg+' '+layer for layer in layers]); vecs_to_plot=[]; file_names=[file_pre+'_'+layer for file_pre in file_pres for layer in layers]; for idx,file_name in enumerate(file_names): print file_name,legend_entries[idx]; [img_paths,gt_labels,indices,_]=pickle.load(open(file_name+'.p','rb')); no_correct,_=script_nearestNeigbourExperiment.getNumberOfCorrectNNMatches(indices,gt_labels) with open(file_name+'.txt','wb') as f: for num in no_correct: f.write(str(num)+' '); f.write('\n'); # with open(file_name+'.txt','rb') as f: # no_correct=f.readline(); # no_correct=[float(no_correct_curr) for no_correct_curr in no_correct.strip('\n').split()]; vecs_to_plot.append(no_correct); # print legend_entries; print vecs_to_plot; # print len(vecs_to_plot); # print len(legend_entries); plt.figure(); plt.xlabel('Number of Nearest Neighbours K'); plt.ylabel('Accuracy'); plt.title('NN Accuracy DNN Features for Pascal Classes'); plt.xlim(0,6); plt.ylim(min([min(vec) for vec in vecs_to_plot])-0.05,max([max(vec) for vec in vecs_to_plot])+0.05); handles=[]; for vec in vecs_to_plot: handle,=plt.plot(range(1,len(vec)+1),vec); handles.append(handle); plt.legend(handles, legend_entries,loc=2,prop={'size':10}); plt.savefig(out_file); legend_entries=['Trained-Untrained '+layer for layer in layers]; diffs=[]; for idx in range(3): a=vecs_to_plot[idx]; b=vecs_to_plot[idx+3]; diff=[a[idx_curr]-b[idx_curr] for idx_curr in range(len(a))]; diffs.append(diff); vecs_to_plot=diffs; plt.figure(); plt.xlabel('Number of Nearest Neighbours K'); plt.ylabel('Accuracy Difference'); plt.title('NN Accuracy DNN Features for Pascal Classes'); plt.xlim(0,6); plt.ylim(min([min(vec) for vec in vecs_to_plot])-0.01,max([max(vec) for vec in vecs_to_plot])+0.01); handles=[]; for vec in vecs_to_plot: handle,=plt.plot(range(1,len(vec)+1),vec); handles.append(handle); plt.legend(handles, legend_entries,loc=2,prop={'size':10}); plt.savefig(out_file_diff);
def script_runNNOnPascalIncludedInTraining(): path_to_file = '../../data/ilsvrc12/synset_words.txt' val_ids = imagenet.readLabelsFile(path_to_file) val_just_ids = list(zip(*val_ids)[0]) val_just_labels = list(zip(*val_ids)[1]) pascal_ids_file = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_classes.txt' pascal_ids = imagenet.readLabelsFile(pascal_ids_file) pascal_just_ids = list(zip(*pascal_ids)[0]) to_exclude = imagenet.removeClassesWithOverlap(val_just_ids, pascal_just_ids, keepMapping=True) val_gt_file = '../../data/ilsvrc12/val.txt' list_of_ids_im = [id for id_list in to_exclude for id in id_list] mapping_file = '../../data/ilsvrc12/synsets.txt' list_of_ids = imagenet.getImagenetIdToTrainingIdMapping( mapping_file, list_of_ids_im) print len(list_of_ids) list_of_ids_pascal = [] for id_no in range(len(to_exclude)): list_of_ids_pascal = list_of_ids_pascal + [id_no] * len( to_exclude[id_no]) path_to_val = '/disk2/imagenet/val' test_set = imagenet.selectTestSetByID(val_gt_file, list_of_ids, path_to_val) out_dir = '/disk2/octoberExperiments/nn_performance_without_pascal/trained' layers = ['pool5', 'fc6', 'fc7'] gpu_no = 0 path_to_classify = '..' numberOfN = 5 relativePaths = ['/disk2', '../../../..'] # out_file=script_nearestNeigbourExperiment.runClassificationTestSet(test_set,out_dir,path_to_classify,gpu_no,layers) file_name = '/disk2/octoberExperiments/nn_performance_without_pascal/trained/20151023153522' file_text_labels = '../../data/ilsvrc12/synset_words.txt' text_labels = np.loadtxt(file_text_labels, str, delimiter='\t') vals = np.load(file_name + '.npz') test_set = sorted(test_set, key=lambda x: x[0]) test_set = zip(*test_set) img_paths = list(test_set[0]) gt_labels = list(test_set[1]) gt_labels_pascal = [ list_of_ids_pascal[list_of_ids.index(gt_label)] for gt_label in gt_labels ] # for layer in layers: # file_name_l=file_name+'_'+layer; # indices,conf_matrix=doNN(img_paths,gt_labels,vals[layer],numberOfN=numberOfN,distance='cosine',algo='brute') # pickle.dump([img_paths,gt_labels,indices,conf_matrix],open(file_name_l+'.p','wb')); file_text_labels_pascal = '/disk2/octoberExperiments/nn_performance_without_pascal/pascal_classes.txt' text_labels_pascal = np.loadtxt(file_text_labels_pascal, str, delimiter='\t') for layer in layers: print layer file_name_l = file_name + '_' + layer [img_paths, gt_labels, indices, _] = pickle.load(open(file_name_l + '.p', 'rb')) img_paths_curr = [ x.replace(relativePaths[0], relativePaths[1]) for x in img_paths ] im_paths, captions = script_nearestNeigbourExperiment.createImageAndCaptionGrid( img_paths_curr, gt_labels, indices, text_labels) script_nearestNeigbourExperiment.writeHTML(file_name_l + '.html', im_paths, captions) no_correct, _ = script_nearestNeigbourExperiment.getNumberOfCorrectNNMatches( indices, gt_labels) print no_correct with open(file_name_l + '.txt', 'wb') as f: for no_correct_curr in no_correct: f.write(str(no_correct_curr) + ' ') file_name_l = file_name + '_' + layer + '_pascal' im_paths, captions = script_nearestNeigbourExperiment.createImageAndCaptionGrid( img_paths_curr, gt_labels_pascal, indices, text_labels_pascal) script_nearestNeigbourExperiment.writeHTML(file_name_l + '.html', im_paths, captions) no_correct, _ = script_nearestNeigbourExperiment.getNumberOfCorrectNNMatches( indices, gt_labels_pascal) with open(file_name_l + '.txt', 'wb') as f: for no_correct_curr in no_correct: f.write(str(no_correct_curr) + ' ') print no_correct