def random(): # Start VGG sess = tf.Session() imgs = tf.placeholder(tf.float32, [None, 224, 224, 3]) vgg = vgg16(imgs, COMMAND_PATH + 'vgg16_weights.npz', sess) # Get random crop list folder = COMMAND_PATH + DATA_PATH + settings.folder_name(DATATYPE) rand_crops = os.listdir(folder) # Get true value image_id = int(sys.argv[2]) image_tag = settings.get_ind_name(image_id) labels_file = open(COMMAND_PATH + 'caffe_ilsvrc12/' + settings.DATASET + '-labels.json') true_labels = json.load(labels_file) true_value = true_labels[image_tag] # Get old classifications if they exist results_folder = COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_randind_results/' old_results_name = settings.get_random_crops_results_name(image_id) if old_results_name in os.listdir(results_folder): with open(results_folder + old_results_name, 'r') as old_results_file: old_crop_classifications = json.load(old_results_file) else: old_crop_classifications = {} # Classify crops and store results in dictionary with open(COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_randind_results/' + image_tag + '_' + 'dim_ids.json', 'r') as dim_ids_file: dim_ids = json.load(dim_ids_file) crop_classifications = {} beginning = 'ILSVRC2012_val_00000000_crop' end = '.JPEG' for rand_crop in rand_crops: crop_id = rand_crop[len(beginning):-1 * len(end)] # from the end of 'crop' to beginning of '.JPEG'. TODO fix; super abusive img1 = imread(folder + rand_crop, mode='RGB') img1 = imresize(img1, (224, 224)) prob = sess.run(vgg.probs, feed_dict={vgg.imgs: [img1]})[0] max_prob = np.amax(prob) pred = (np.argsort(prob))[-1] print 'PRED:', pred print 'PROB:', max_prob dims = dim_ids[crop_id] if pred == true_value and max_prob > settings.CROP_CONFIDENCE_THRESHOLD: crop_classifications[str(dims)] = True else: crop_classifications[str(dims)] = False # Update old classifications with new ones and save everything old_crop_classifications.update(crop_classifications) with open(results_folder + image_tag + '_crop_results.json', 'w') as results_file: json.dump(old_crop_classifications, results_file) print old_crop_classifications shutil.rmtree(folder)
def get_crop_size(smalldataset_id, crop_metric): imagenetval_id = settings.convert_id_small_to_imagenetval(smalldataset_id) image_tag = settings.get_ind_name(imagenetval_id) image_filename = PATH_TO_DATA + settings.folder_name('img') + image_tag + '.JPEG' im = Image.open(image_filename) width, height = im.size # print('IMAGE SHAPE:', width, height) crop_type = 'proportional' if crop_metric <= 1. else 'constant' crop_size = c_m_p.get_crop_size(height, crop_metric, crop_type) if height <= width else c_m_p.get_crop_size(width, crop_metric, crop_type) return crop_size
def save_crops(coords, crop_metric, model_name, image_scale, axis, compare_corr, num_samples=50): folder = PATH_TO_OUTPUT_DATA + settings.maxdiff_folder_name( axis, crop_metric, model_name, image_scale, 'diff' if compare_corr else 'any') if not os.path.exists(folder): os.makedirs(folder) # top_ids = sorted(coords, reverse=True, key=lambda x: coords[x][2])[num_samples:num_samples * 2] # print(top_ids) for smalldataset_id in coords: print('CURRENTLY SAVING:', smalldataset_id) im_filename = PATH_TO_DATA + settings.folder_name( 'img') + settings.get_ind_name( settings.convert_id_small_to_imagenetval( smalldataset_id)) + '.JPEG' im = Image.open(im_filename) hcell, lcell = coords[smalldataset_id][:2] hfn, lfn = (PATH_TO_OUTPUT_DATA + settings.maxdiff_file_name( smalldataset_id, axis, crop_metric, model_name, image_scale, 'diff' if compare_corr else 'any', conf) for conf in ['high', 'low']) if axis == 'scale': high_size = get_crop_size(smalldataset_id, crop_metric) low_size = high_size - 2 hcrop = im.crop((hcell[0], hcell[1], hcell[0] + high_size, hcell[1] + high_size)) lcrop = im.crop( (lcell[0], lcell[1], lcell[0] + low_size, lcell[1] + low_size)) elif axis == 'shift': size = get_crop_size(smalldataset_id, crop_metric) hcrop = im.crop( (hcell[0], hcell[1], hcell[0] + size, hcell[1] + size)) lcrop = im.crop( (lcell[0], lcell[1], lcell[0] + size, lcell[1] + size)) print(hfn) print(lfn) hcrop.save(hfn, 'JPEG') lcrop.save(lfn, 'JPEG') print('SAVES COMPLETE')
def folder_name(datatype): ''' Parameters ---------- datatype (str): the type of data, e.g. crop or backblk Returns ------- folder name with command path and data path attached, because all functions here deal with data files ''' return DATA_PATH + settings.folder_name(datatype)
def minimal_image_distribution(num_imgs, crop_metric, model_name, image_scale, strictness): resize_dim = 150 minimal_image_aggregation = np.zeros((resize_dim, resize_dim)) # img_ids = random.sample(range(100), num_imgs) # for testing on my machine: only a subset of the maps. TODO remove for full job img_ids = range(3) for smalldataset_id in img_ids: # get bbx dimensions imagenetval_id = settings.convert_id_small_to_imagenetval(smalldataset_id) image_tag = settings.get_ind_name(imagenetval_id) with open(BBX_FILE, 'r') as bbx_file: all_bbxs = json.load(bbx_file) crop_dims = [bbx[0] for bbx in all_bbxs[image_tag]] # get all x1, y1, x2, y2 crops minimal_map_f = PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, smalldataset_id) minimal_map_f = minimal_map_f + '_' + ('l' if strictness == 'loose' else '') + 'map' minimal_map = np.load(minimal_map_f + '.npy') image_filename = PATH_TO_DATA + settings.folder_name('img') + image_tag + '.JPEG' try: # for testing on my machine: if the image is not on my machine, move on. TODO remove for full job im = Image.open(image_filename) except OSError: continue width, height = im.size crop_type = 'proportional' if crop_metric <= 1. else 'constant' crop_size = c_m_p.get_crop_size(height, crop_metric, crop_type) if height <= width else c_m_p.get_crop_size(width, crop_metric, crop_type) for x1, y1, x2, y2 in crop_dims: minmap_sub = minimal_map[y1:y2 - crop_size + 1, x1:x2 - crop_size + 1] minmap_sub = imresize(minmap_sub, (resize_dim, resize_dim)) minimal_image_aggregation += minmap_sub vis = (minimal_image_aggregation - np.min(minimal_image_aggregation)) vis /= np.max(vis) vis *= 255. Image.fromarray(vis).show() return minimal_image_aggregation
def backprop_crops(image_id, crop_metric, model_name, image_scale): ''' note: even with the graph inefficiency, it makes more sense for this function to be one image at a time because the indices are different for each one. This would only change if I figured out a programmatic way to find the right indices. ''' # Get indices for this map with open(PATH_TO_DATA + settings.BACKPROP_INDICES_FILENAME, 'r') as indfile: all_inds = json.load(indfile) indices = all_inds[str( (crop_metric, model_name, image_scale, image_id))] with tf.Session() as sess: # Set up CNN model = settings.MODELS[model_name] batch_size = len(indices) imgs = tf.placeholder(tf.float32, [batch_size, model.im_size, model.im_size, 3]) network = model(imgs, sess) # Create backprop objects true_labels = json.load( open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json')) true_label = true_labels[settings.get_ind_name(image_id)] y = tf.constant([true_label for __ in range(batch_size)]) err = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=y, logits=network.logits) dy_dx = tf.gradients(err, network.imgs) # Scale image, get all crops and run backprop image = Image.open(PATH_TO_DATA + settings.folder_name('img') + settings.get_ind_name(image_id) + '.JPEG') width, height = image.size image = imresize(image, (int(width * image_scale), int(height * image_scale))) crop_type = 'proportional' if crop_metric <= 1. else 'constant' crop_size = get_crop_size( height, crop_metric, crop_type) if height <= width else get_crop_size( width, crop_metric, crop_type) all_crops = [] for x1, y1 in indices: crop = image[y1:y1 + crop_size, x1:x1 + crop_size, :] # crop = image.crop((x1, y1, x1 + crop_size, y1 + crop_size)) all_crops.append(imresize(crop, (model.im_size, model.im_size))) # all_crops.append(imresize(image, (model.im_size, model.im_size))) # TODO remove after making sure this code is working backprops = sess.run( dy_dx, feed_dict={network.imgs: model.preprocess(np.array(all_crops))})[0] # Make backprop results visualizable backprops = backprops - np.min(backprops, axis=(1, 2), keepdims=True) backprops = backprops / np.max(backprops, axis=(1, 2), keepdims=True) backprops = backprops * 255. backprops = backprops.astype(np.uint8) # savez results folder = PATH_TO_DATA + settings.map_folder_name( settings.BACKPROP_MAPTYPE, crop_metric, model_name, image_scale) filename = PATH_TO_DATA + settings.map_filename( settings.BACKPROP_MAPTYPE, crop_metric, model_name, image_scale, image_id) if not os.path.exists(folder): os.makedirs(folder) np.savez(filename, **{str(indices[i]): backprops[i] for i in range(len(indices))})
def get_max_diff_adjacent_crops( start_id, end_id, crop_metric, model_name, image_scale, compare_corr=True, use_top5=True ): # eventually add step size, right now defaults to 1 in code # compare_correctness: bool indicating whether the final crops should necessarily have different classification correctness # use_top5: bool indicating whether using top5 correctness or top1 correctness # returns the two crops. If compare_correctness==True and there are no correctly classified crops, returns None. crop_type = 'proportional' if crop_metric <= 1. else 'constant' folders = [ PATH_TO_DATA + settings.maxdiff_folder_name( 'size', crop_metric, model_name, image_scale, 'diff' if compare_corr else 'any', conf) for conf in ['high', 'low'] ] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) sess = tf.Session() model = settings.MODELS[model_name] imgs = tf.placeholder(tf.float32, [None, model.im_size, model.im_size, 3]) network = model(imgs, sess) image_ids = range(start_id, end_id + 1) hc_correct = [] hc_incorrect = [] lc_correct = [] lc_incorrect = [] h_activations = {} l_activations = {} for image_id in image_ids: f = open('small-dataset-to-imagenet.txt') lines = f.readlines() image_tag = lines[image_id].split(" ", 1)[0] print(image_tag) #image_tag = settings.get_ind_name(image_id) image_filename = PATH_TO_DATA + settings.folder_name( 'img') + image_tag #+ ('.png' if image_id == 50001 else '.JPEG') image = Image.open(image_filename) if image.mode != 'RGB': image = image.convert('RGB') image = np.asarray(image) image_label = image_tag[:-5] + '_' + str( crop_metric) + '_' + model_name + '_' correctness_type = 'top5' if use_top5 else 'top1' correctness_filename = PATH_TO_DATA + (settings.map_folder_name( settings.TOP5_MAPTYPE, crop_metric) if use_top5 else settings.map_folder_name( settings.TOP1_MAPTYPE, crop_metric)) + image_label + ( settings.TOP5_MAPTYPE if use_top5 else settings.TOP1_MAPTYPE) + '.npy' cor_map = np.load(correctness_filename) corr_extension = '_diffcorrectness' if compare_corr else '_anycorrectness' if compare_corr: if not cor_map.any(): print('%s has no correctly classified crops.' % image_tag) continue elif cor_map.all(): print('%s has only correctly classified crops.' % image_tag) continue con_map_filename = PATH_TO_DATA + settings.map_folder_name( settings.CONFIDENCE_MAPTYPE, crop_metric) + image_label + settings.CONFIDENCE_MAPTYPE + '.npy' con_map = np.load(con_map_filename) down_diff = np.diff( con_map, axis=0) # apparently assumes step_size=1 (adjacency) up_diff = -1. * down_diff right_diff = np.diff(con_map) left_diff = -1. * right_diff diffs = { 'up': up_diff, 'down': down_diff, 'left': left_diff, 'right': right_diff } # TESTER: CLASSIFYING CROPS TO SEE IF THERE'S A DIFFERENCE BETWEEN WHAT THE TESTER REPORTS AND WHAT THIS REPORTS true_labels = json.load( open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json')) while True: maxes = { direction: np.unravel_index(np.argmax(diffs[direction]), diffs[direction].shape) for direction in diffs } # map each directional diff to its argmax (index of its maximum confidence diff) max_dir = max( [direction for direction in maxes], key=lambda direction: diffs[direction][tuple(maxes[direction])] ) # get the direction of the diff whose max confidence is the highest out of the four max confidences # depending on the max-confidence direction, get the argmax of that direction. The more confident crop will be offset by 1 in a way that depends on the direction. if max_dir == 'up': up_max = maxes['up'] gcell, cell = ( tuple(up_max), (up_max[0] + 1, up_max[1]) ) # up (and left) are like this because when up and left diffs are made, the negation also changes the direction in which your step goes. it goes down -> up; right -> left. elif max_dir == 'down': down_max = maxes['down'] cell, gcell = (tuple(down_max), (down_max[0] + 1, down_max[1])) elif max_dir == 'left': left_max = maxes['left'] gcell, cell = (tuple(left_max), (left_max[0], left_max[1] + 1)) else: right_max = maxes['right'] cell, gcell = (tuple(right_max), (right_max[0], right_max[1] + 1)) diff_correctness = cor_map[cell] != cor_map[gcell] if diff_correctness or not compare_correctness: y, x = cell gy, gx = gcell height, width, channels = image.shape crop_size = get_crop_size( height, proportion) if height <= width else get_crop_size( width, proportion) dim_used = 'height' if height <= width else 'width' cropped = image[cell[0]:cell[0] + crop_size, cell[1]:cell[1] + crop_size] gcropped = image[gcell[0]:gcell[0] + crop_size, gcell[1]:gcell[1] + crop_size] true_value = true_labels[image_tag[:-5]] hc = imresize(gcropped, (network.im_size, network.im_size)) lc = imresize(cropped, (network.im_size, network.im_size)) hresult = sess.run(network.pull_layers, feed_dict={network.imgs: [hc]}) hcprob = hresult[0][0] h_activations[image_id] = hresult[1:] result = sess.run(network.pull_layers, feed_dict={network.imgs: [lc]}) lcprob = lresult[0][0] l_activations[image_id] = lresult[1:] hcpreds = (np.argsort(hcprob)[::-1])[0:5] lcpreds = (np.argsort(lcprob)[::-1])[0:5] if true_value in hcpreds: hc_correct.append(image_id) else: hc_incorrect.append(image_id) if true_value in lcpreds: lc_correct.append(image_id) else: lc_incorrect.append(image_id) maxdiff_folder = settings.maxdiff_folder_name(crop_metric) np.save( PATH_TO_DATA + maxdiff_folder + image_label + 'maxdiff_lowconf' + corr_extension, cropped) np.save( PATH_TO_DATA + maxdiff_folder + image_label + 'maxdiff_highconf' + corr_extension, gcropped) break else: if max_dir in ['up', 'left']: diffs[max_dir][ gcell] = -2. # for the diff where that was the argmax, mark the cell containing it to something lower than any real entry (-1. <= real entry <= 1.) This is gcell for up, left and cell for down, right because the lower-indexed cell is always the one that contained the confidence originally else: diffs[max_dir][cell] = -2. print('INTERNAL TEST') print("High confidence crop correctly classified:", hc_correct) print('High confidence crop incorrectly classified:', hc_incorrect) print('Low confidence crop correctly classified:', lc_correct) print('Low confidence crop incorrectly classified:', lc_incorrect) sess.close() return h_activations, l_activations
def get_maxdiff_size_crops(start_id, end_id, crop_metric, model_name, image_scale, compare_corr=True): # For each image id in range(start_id, end_id + 1), finds the crop of size crop_metric and the crop ~2 pixels smaller that are maximally different in confidence. If compare_corr is True, it necessarily finds crops where the smaller one is classified incorrectly and the larger one is classified correctly. This uses the top5 maps for the two scales. crop_type = 'proportional' if crop_metric <= 1. else 'constant' folders = [ PATH_TO_DATA + settings.maxdiff_folder_name( 'size', crop_metric, model_name, image_scale, 'diff' if compare_corr else 'any', conf) for conf in ['high', 'low'] ] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) map_folder = PATH_TO_DATA + settings.maxdiff_folder_name( 'size', crop_metric, model_name, image_scale, 'map') if not os.path.exists(map_folder): os.makedirs(map_folder) # with tf.Session() as sess: # TODO use for testing # model = settings.MODELS[model_name] # imgs = tf.placeholder(tf.float32, [None, model.im_size, model.im_size, 3]) # network = model(imgs, sess) true_labels = json.load( open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json')) for image_id in range(start_id, end_id + 1): image_tag = settings.get_ind_name(image_id) image_filename = PATH_TO_DATA + settings.folder_name( 'img') + image_tag + '.JPEG' true_class = true_labels[image_tag] im = Image.open(image_filename) if im.mode != 'RGB': im = im.convert('RGB') width, height = im.size im = im.resize((int(width * image_scale), int(height * image_scale))) width, height = im.size im = np.asarray(im) # Get the small crop_metric, crop sizes for the large and small crop_metrics size_dim = height if height <= width else width large_size = get_crop_size( size_dim, crop_metric, crop_type) if height <= width else get_crop_size( width, crop_metric, crop_type) small_metric = 0.194 if crop_metric == 0.2 else 0.394 # TODO change to be a calculation and command small_size = get_crop_size(size_dim, small_metric, crop_type) metrics = [crop_metric, small_metric] # Get the correctness maps (top5, may become a choice between top5 and top1 in the future), and if the call requires diff correctness, check that that's possible corr_fns = [ PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, metric, model_name, image_scale, image_id) + '.npy' for metric in metrics ] # TODO change to allow choice of top1 or top5 lcor, scor = cor_maps = [np.load(corr_fn) for corr_fn in corr_fns] if compare_corr: for cor_map in cor_maps: if not cor_map.any(): print('%s has no correctly classified crops.' % image_tag) continue elif cor_map.all(): print('%s has only correctly classified crops.' % image_tag) continue # Get confidence maps con_fns = [ PATH_TO_DATA + settings.map_filename(settings.CONFIDENCE_MAPTYPE, metric, model_name, image_scale, image_id) + '.npy' for metric in metrics ] lcon, scon = [np.load(con_fn) for con_fn in con_fns] # Calculate difference matrices lrows, lcols = lcon.shape offset = large_size - small_size # get the metric that bottom and right are off by tl_sub = scon[:lrows, : lcols] # for top left, get the top left small crops that correspond to big crops - same shape. It's all of them because the large crops are all adjacent, even if the difference in pixels is >1. tr_sub = scon[:lrows, offset:lcols + offset] # for top right, everything that is 'offset' cols over bl_sub = scon[offset:lrows + offset, :lcols] br_sub = scon[offset:lrows + offset, offset:lcols + offset] ctoffset = int(offset / 2) ct_sub = scon[ctoffset:lrows + ctoffset, ctoffset:lcols + ctoffset] diffs = { 'tl': lcon - tl_sub, # use subtraction because we are looking for increase in conf from increase in size 'tr': lcon - tr_sub, 'bl': lcon - bl_sub, 'br': lcon - br_sub, 'ct': lcon - ct_sub } # Make map of the largest size change in confidence across all directions of shrinking change_map = np.maximum.reduce(list(diffs.values())) np.save(map_folder + str(image_id), change_map) # Find maxdiff pair by searching for maximally different pairs until one with different correctness is found (if diffcor. Else, this will terminate after one loop as the first pair found will be the maximally different one and therefore the right one for anycor.) while True: maxes = { corner: np.unravel_index(np.argmax(diffs[corner]), diffs[corner].shape) for corner in diffs } # map each corner diff to its argmax (index of maximum confidence diff) max_dir = max( [corner for corner in maxes], key=lambda corner: diffs[corner][tuple(maxes[corner])] ) # get the corner id of the diff whose max change is the highest out of the four max changes # getting the indices of the maximal confidence increase. Indices are based on the size of the large crop size map. The first index is the index for the large crop, and the second is for the small crop. corner_max = maxes[max_dir] if max_dir == 'tl': lcell, scell = tuple(corner_max), tuple(corner_max) elif max_dir == 'tr': lcell, scell = tuple(corner_max), (corner_max[0], corner_max[1] + offset) elif max_dir == 'bl': lcell, scell = tuple(corner_max), (corner_max[0] + offset, corner_max[1]) elif max_dir == 'br': lcell, scell = tuple(corner_max), (corner_max[0] + offset, corner_max[1] + offset) else: lcell, scell = tuple(corner_max), (corner_max[0] + ctoffset, corner_max[1] + ctoffset) diff_corr = lcor[lcell] != scor[scell] if diff_corr or not compare_corr: sy, sx = scell ly, lx = lcell lcropped = im[lcell[0]:lcell[0] + large_size, lcell[1]:lcell[1] + large_size] scropped = im[scell[0]:scell[0] + small_size, scell[1]:scell[1] + small_size] # lcropped = imresize(lcropped, (network.im_size, network.im_size)) # scropped = imresize(scropped, (network.im_size, network.im_size)) # result = sess.run(network.probs, feed_dict={network.imgs: np.array([lcropped, scropped])}) # run and see if it works later. Without this, the sess isn't actually being used - this is for internal test. break else: # if that location wasn't diffcorr, set the diff's entry to -2. diffs[max_dir][lcell] = -2. lfolder, sfolder = folders np.save(lfolder + str(image_id), lcropped) np.save(sfolder + str(image_id), scropped)
def test_maxdiff_crops(start_id, end_id, crop_metric, model_name='vgg16'): # crop_metric = proportion if CROP_TYPE == 'proportional' else CONSTANT sess = tf.Session() model = settings.MODELS[model_name] imgs = tf.placeholder(tf.float32, [None, model.im_size, model.im_size, 3]) network = model(imgs, sess) true_labels = json.load( open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json')) image_ids = range(start_id, end_id + 1) hc_incorrect = [] lc_incorrect = [] hc_correct = [] lc_correct = [] h_activations = {} l_activations = {} for image_id in image_ids: f = open('small-dataset-to-imagenet.txt') lines = f.readlines() image_tag = lines[image_id].split(" ", 1)[0] print(image_tag) #image_tag = settings.get_ind_name(image_id) image_label = image_tag[:-5] + '_' + str( crop_metric ) + '_' # TODO I think I got ahead of myself and put in model name to this function when maxdiff crops haven't been adjusted, but eventually, fix this with the new filesystem true_value = true_labels[image_tag[:-5]] maxdiff_folder = settings.maxdiff_folder_name(crop_metric) try: highconf_filename = PATH_TO_DATA + maxdiff_folder + image_label + 'maxdiff_highconf_diffcorrectness.npy' highconf_crop = np.load(highconf_filename) except FileNotFoundError: print(image_tag, 'does not have diff-correctness crops') continue lowconf_filename = PATH_TO_DATA + maxdiff_folder + image_label + 'maxdiff_lowconf_diffcorrectness.npy' lowconf_crop = np.load(lowconf_filename) y = tf.constant([true_value]) err = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=y, logits=network.fc3l) # TODO adapt to other models beyond VGG dy_dx = tf.gradients(err, network.imgs) hc = imresize(highconf_crop, (network.im_size, network.im_size)) lc = imresize(lowconf_crop, (network.im_size, network.im_size)) hcresult = sess.run([network.probs, dy_dx], feed_dict={network.imgs: [hc]}) lcresult = sess.run([network.probs, dy_dx], feed_dict={network.imgs: [lc]}) hcprob = hcresult[0][0] lcprob = lcresult[0][0] hcpreds = (np.argsort(hcprob)[::-1])[0:5] lcpreds = (np.argsort(lcprob)[::-1])[0:5] if true_value in hcpreds: hc_correct.append(image_id) else: hc_incorrect.append(image_id) if true_value in lcpreds: lc_correct.append(image_id) else: lc_incorrect.append(image_id) h_activations[image_id] = hcresult[1:] l_activations[image_id] = lcresult[1:] hgrad = np.squeeze(hcresult[1][0]) lgrad = np.squeeze(lcresult[1][0]) hgrad = hgrad - hgrad.min() lgrad = lgrad - lgrad.min() hgrad = hgrad / hgrad.max() lgrad = lgrad / lgrad.max() hgrad = hgrad * 255. lgrad = lgrad * 255. hgrad = hgrad.astype(np.uint8) lgrad = lgrad.astype(np.uint8) print('H Shape:', hgrad.shape) print('L Shape:', lgrad.shape) hgrad_vis = Image.fromarray(hgrad, mode='RGB') lgrad_vis = Image.fromarray(lgrad, mode='RGB') hgrad_vis.save(PATH_TO_DATA + str(image_id) + 'hgrad_vis.JPEG', 'JPEG') lgrad_vis.save(PATH_TO_DATA + str(image_id) + 'lgrad_vis.JPEG', 'JPEG') # SANITY CHECK for backprop code image_filename = PATH_TO_DATA + settings.folder_name( 'img') + image_tag #+ ('.png' if image_id == 50001 else'.JPEG') im = imread(image_filename, mode='RGB') im = imresize(im, (network.im_size, network.im_size)) fullresult = sess.run([network.probs, dy_dx], feed_dict={network.imgs: [im]}) fullgrad = np.squeeze(fullresult[1][0]) fullgrad = fullgrad - fullgrad.min() fullgrad = fullgrad / fullgrad.max() fullgrad = fullgrad * 255. fullgrad = fullgrad.astype(np.uint8) # np.uint8) print('OG SHAPE:', im.shape) print('FG SHAPE:', fullgrad.shape) print('EQUIVALENT:', np.array_equal(im, fullgrad)) print('OG TYPE:', type(im[0][0][0])) print('FG TYPE:', type(fullgrad[0][0][0])) debug_folder = 'debugging/' original = Image.fromarray(im, mode='RGB') original.save(debug_folder + 'og.JPEG', 'JPEG') print('FULLGRAD:', fullgrad) fullgrad_vis = Image.fromarray(fullgrad, mode='RGB') fullgrad_vis.save(debug_folder + 'fg.JPEG', 'JPEG') print('EXTERNAL TEST') print("High confidence crop correctly classified:", hc_correct) print('High confidence crop incorrectly classified:', hc_incorrect) print('Low confidence crop correctly classified:', lc_correct) print('Low confidence crop incorrectly classified:', lc_incorrect) return h_activations, l_activations
def create_confidence_map(start_id, end_id, crop_metric, model_name, image_scale, make_cmap=True, make_top5=True, make_top1=True, make_distance=True): # Get the crop type - if the crop_metric is a fraction, it's a proportion. If it's larger than 1, it's a constant crop size. crop_type = 'proportional' if crop_metric <= 1. else 'constant' # Prepare folders maptypes = [ settings.CONFIDENCE_MAPTYPE, settings.TOP5_MAPTYPE, settings.TOP1_MAPTYPE, settings.DISTANCE_MAPTYPE ] folders = [ PATH_TO_DATA + settings.map_folder_name(maptype, crop_metric, model_name, image_scale) for maptype in maptypes ] for folder in folders: if not os.path.exists(folder): os.makedirs(folder) # Get the right model model = settings.MODELS[model_name] # Before anything else happens, so we only set up once despite multiple images, make a network for each GPU config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) networks = [] for i in range(NUM_GPUS): gpu = 'device:GPU:%d' % i with tf.device(gpu): imgs = tf.placeholder( tf.float32, [BATCH_SIZE, model.im_size, model.im_size, 3]) if not model_name == 'alexnet': network = model(imgs, sess, reuse=None if i == 0 else True) else: network = model(imgs, sess) networks.append(network) # Get each map called for! labels_file = open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json') true_labels = json.load(labels_file) image_ids = range(start_id, end_id + 1) for image_id in image_ids: f = open('small-dataset-to-imagenet.txt') lines = f.readlines() image_tag = lines[image_id].split(" ", 1)[0] print(image_tag) #image_tag = settings.get_ind_name(image_id) image_filename = PATH_TO_DATA + settings.folder_name( 'img') + image_tag #+ ('.png' if image_id == 50001 else'.JPEG') true_class = true_labels[image_tag[:-5]] if image_id != 50001 else 266 im = Image.open(image_filename) if im.mode != 'RGB': im = im.convert( 'RGB' ) # make sure bw images are 3-channel, because they get opened in L mode width, height = im.size # Resize image based on image_scale im = im.resize((int(width * image_scale), int(height * image_scale))) width, height = im.size # TODO do resizing experiment im = imresize(im, (width*image_scale, height*image_scale)) # resize image as needed crop_size = get_crop_size( height, crop_metric, crop_type) if height <= width else get_crop_size( width, crop_metric, crop_type) crop_size -= 2 # reduce size by two pixels for small crops crop_x1s = range(width - crop_size + 1) crop_y1s = range(height - crop_size + 1) crop_dims = itertools.product(crop_x1s, crop_y1s) C, O, F, D = (np.zeros((height - crop_size + 1, width - crop_size + 1)) for __ in range(4)) crop_dims = list(crop_dims) total_crops = len(crop_dims) crop_index = 0 overall_start_time = time.clock() print('TOTAL CROPS:', total_crops) sys.stdout.flush() while crop_index < total_crops: # While we haven't exhausted crops TODO see if the logic needs to be changed all_cropped_imgs = [] stub_crops = 0 # Initializing to 0 in case there's an image with no stub crops (i.e. number of crops is a multiple of 64) map_indices = [ ] # The indices that these confidences will be mapped to for i in range( BATCH_SIZE * NUM_GPUS ): # Get BATCH_SIZE crops for each GPU (total of NUM_GPUS) if crop_index == total_crops: # If we're on the last round and it's not all 64, repeat the last crop for the rest stub_crops = ( BATCH_SIZE * NUM_GPUS ) - i # the number of crops still needed, because i in this round is the number of crops that have already been filled in cropped = imresize( cropped, (model.im_size, model.im_size) ) # resize the last one (from previous round) permanently. There will be a previous one due to the while logic for i in range(stub_crops): all_cropped_imgs.append( cropped ) # fill in the rest of the slots with the resized last crop break # If not on the last one, continue on to fill in the next crop x1, y1 = crop_dims[crop_index] map_indices.append((x1, y1)) cropped = im.crop((x1, y1, x1 + crop_size, y1 + crop_size)) all_cropped_imgs.append( imresize(cropped, (model.im_size, model.im_size))) crop_index += 1 # Increment to get the next crop dimensions start_time = time.clock() network_probs = list(map(lambda x: x.probs, networks)) num_crops = len(all_cropped_imgs) partition_cropped_imgs = [ all_cropped_imgs[int(i * num_crops / NUM_GPUS):min( num_crops, int((i + 1) * num_crops / NUM_GPUS))] for i in range(NUM_GPUS) ] # TODO yikes is this correct prob = np.array( sess.run(network_probs, feed_dict={ networks[i].imgs: model.preprocess( np.array(partition_cropped_imgs[i])) for i in range(NUM_GPUS) })) # prob = sess.run(vgg.probs, feed_dict={vgg.imgs: all_cropped_imgs}) end_time = time.clock() print('Time for running one size-' + str(BATCH_SIZE), 'batch:', end_time - start_time, 'seconds') print('CROPS COMPLETED SO FAR:', crop_index) sys.stdout.flush() # plot the confidences in the map. For final iteration, which likely has <BATCH_SIZE meaningful crops, the index list being of shorter length will cause them to be thrown. confidence = prob[:, :, true_class].reshape(BATCH_SIZE * NUM_GPUS) for i in range(len(map_indices)): c, r = map_indices[i] C[r, c] = confidence[i] # plot the top-5 and top-1 binary correctness maps flat_probs = prob.reshape( (NUM_GPUS * BATCH_SIZE, settings.NUM_CLASSES)) sorted_classes = flat_probs.argsort(axis=1) top_5 = sorted_classes[:, -5:] top_1 = sorted_classes[:, -1].squeeze() for i in range(len(map_indices)): c, r = map_indices[i] O[r, c] = 255. if top_1[i] == true_class else 0. F[r, c] = 255. if true_class in top_5[i] else 0. # plot the distance map sixth = sorted_classes[:, 6] for i in range(len(map_indices)): c, r = map_indices[i] if true_class in top_5[i]: D[r, c] = 255. * flat_probs[i][true_class] - flat_probs[i][ sixth[i]] else: D[r, c] = 0. overall_end_time = time.clock() print( 'Time for overall cropping and map-building process with size-' + str(BATCH_SIZE), 'batch:', overall_end_time - overall_start_time, 'seconds') sys.stdout.flush() # Make confidence map, save to confidence map folder if make_cmap: np.save( PATH_TO_DATA + settings.map_filename(settings.CONFIDENCE_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_small', C) # Save binary correctness maps if make_top5: np.save( PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_small', F) if make_top1: np.save( PATH_TO_DATA + settings.map_filename(settings.TOP1_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_small', O) # Save distance maps if make_distance: np.save( PATH_TO_DATA + settings.map_filename(settings.DISTANCE_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_small', D) print(image_tag)
def best_crops(): # Start VGG sess = tf.Session() imgs = tf.placeholder(tf.float32, [None, 224, 224, 3]) vgg = vgg16(imgs, COMMAND_PATH + 'vgg16_weights.npz', sess) # Get best crop records - CHANGED TO just start a new best crop record # results_folder = COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_randind_results/' # if 'best-crops.json' in os.listdir(results_folder): # with open(COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_randind_results/best-crops.json', 'r') as best_crop_file: # best_crop_record = json.load(best_crop_file) # else: # best_crop_record = {} best_crop_record = {} start_id = int(sys.argv[2]) end_id = int(sys.argv[3]) num_crops = int(sys.argv[4]) for image_id in range(start_id, end_id + 1): # Get random crop list folder = folder = COMMAND_PATH + DATA_PATH + settings.folder_name(DATATYPE + '%08d' % image_id) rand_crops = os.listdir(folder) # Get true value image_tag = settings.get_ind_name(image_id) labels_file = open(COMMAND_PATH + 'caffe_ilsvrc12/' + settings.DATASET + '-labels.json') true_labels = json.load(labels_file) true_value = true_labels[image_tag] # Classify crops and get the best crop dim_id_path = COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_randind_results/' + image_tag + '_dim_ids.json' with open(dim_id_path, 'r') as dim_ids_file: dim_ids = json.load(dim_ids_file) best_prob = 0 best_dims = None beginning = 'ILSVRC2012_val_00000000_crop' end = '.JPEG' for rand_crop in rand_crops: crop_id = rand_crop[len(beginning):-1 * len(end)] # from the end of 'crop' to beginning of '.JPEG'. TODO fix; super abusive img1 = imread(folder + rand_crop, mode='RGB') img1 = imresize(img1, (224, 224)) prob = sess.run(vgg.probs, feed_dict={vgg.imgs: [img1]})[0] spec_prob = prob[true_value] # get the probability of this image being the true value if spec_prob > best_prob: # if it's better, save its dimensions and check whether it correctly classified the image best_prob = spec_prob best_dims = dim_ids[crop_id] pred = (np.argsort(prob))[-5:] # get the actual top 5 predictions if true_value in pred: correct = True else: correct = False print 'IMAGE ID:', image_id print type(best_crop_record) print len(best_crop_record) # if str(image_id) not in best_crop_record or float(best_crop_record[str(image_id)][0]) < best_prob: # if it doesn't exist yet or this new probability is better than the old one # best_crop_record[image_id] = (str(best_prob), best_dims, correct) # update the best crop to this best_crop_record[image_id] = (str(best_prob), best_dims, correct) # just add it - no longer updating an existing one os.remove(dim_id_path) # Get rid of the dimensions file shutil.rmtree(folder) # Get rid of the crops # Update test records with an entry on number of images and performance on those images with open(COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_randind_results/best-crops.json', 'w') as best_crop_file: json.dump(best_crop_record, best_crop_file) success = sum(best_crop_record[image_id][2] for image_id in best_crop_record) total = len(best_crop_record) top5_accuracy = float(success)/float(total) fieldnames = settings.DATA_FIELDNAMES with open(COMMAND_PATH + DATA_PATH + 'test_statistics.csv', 'rU') as stats_file: stats_reader = csv.DictReader(stats_file, fieldnames=fieldnames) # TODO make this code less disgusting testID = 0 for line in stats_reader: testID = line['testID'] testID = int(testID) + 1 with open(COMMAND_PATH + DATA_PATH + 'test_statistics.csv', 'a') as stats_file: entry = {'testID': testID, 'imagenet_set': settings.IMAGENET_SET, 'dataset': settings.DATASET, 'datatype': DATATYPE, 'num_images': total, # 'top1_accuracy': None, 'top5_accuracy': top5_accuracy, 'aggregation': None, 'num_crops': num_crops} stats_writer = csv.DictWriter(stats_file, fieldnames=fieldnames) stats_writer.writerow(entry)
def test(): # Run a test on a large set of (potentially transformed) images across all crops with some kind of aggregation num_images = int(sys.argv[3]) sess = tf.Session() imgs = tf.placeholder(tf.float32, [None, 224, 224, 3]) vgg = vgg16(imgs, COMMAND_PATH + 'vgg16_weights.npz', sess) folder = COMMAND_PATH + DATA_PATH + settings.folder_name(DATATYPE) nametags = settings.get_all_names(num_images, DATATYPE) names = os.listdir(folder) total_names = len(names) labels_file = open(COMMAND_PATH + 'caffe_ilsvrc12/' + settings.DATASET + '-labels.json') true_labels = json.load(labels_file) top1_accuracy = 0. top5_accuracy = 0. results = {} name_index = 0 aggregation = sys.argv[2] for image_tag in nametags: probs = [] while image_tag in names[name_index]: # if the current filename contains the image tag image_name = names[name_index] img1 = imread(folder + image_name, mode='RGB') img1 = imresize(img1, (224, 224)) prob = sess.run(vgg.probs, feed_dict={vgg.imgs: [img1]})[0] probs.append(prob) name_index += 1 if name_index >= total_names: break all_probs = np.array(probs) if aggregation == 'avg': prob = np.average(all_probs, 0) else: prob = np.amax(all_probs, 0) preds = (np.argsort(prob)[::-1])[0:5] true_value = true_labels[image_tag] if true_value in preds: top5_accuracy += 1. results[image_tag] = True if true_value == preds[0]: top1_accuracy += 1. else: results[image_tag] = False # for p in preds: # print p # print class_names[p], prob[p] # Save results of individual images (whether classified correctly or not) with open(COMMAND_PATH + DATA_PATH + settings.IMAGENET_SET + '_results/' + settings.IMAGENET_SET + '_' + DATATYPE + '_' + settings.DATASET + '_results' + '.json', 'w') as results_file: json.dump(results, results_file) # Print accuracies and save in test statistics CSV top1_accuracy /= num_images top5_accuracy /= num_images fieldnames = settings.DATA_FIELDNAMES with open(COMMAND_PATH + DATA_PATH + 'test_statistics.csv', 'rU') as stats_file: stats_reader = csv.DictReader(stats_file, fieldnames=fieldnames) # TODO make this code less disgusting testID = 0 for line in stats_reader: testID = line['testID'] testID = int(testID) + 1 with open(COMMAND_PATH + DATA_PATH + 'test_statistics.csv', 'a') as stats_file: entry = {'testID': testID, 'imagenet_set': settings.IMAGENET_SET, 'dataset': settings.DATASET, 'datatype': DATATYPE, 'num_images': num_images, 'top1_accuracy': top1_accuracy, 'top5_accuracy': top5_accuracy, 'aggregation': aggregation, 'num_crops': None} stats_writer = csv.DictWriter(stats_file, fieldnames=fieldnames) stats_writer.writerow(entry) print entry if not DATATYPE == 'img': # We don't want to remove the images, but everything else yes shutil.rmtree(folder) print 'TOP 1 ACCURACY:', top1_accuracy print 'TOP 5 ACCURACY:', top5_accuracy
import sys import tensorflow as tf import settings # DATA PARAMS PATH_TO_DATA = '../../poggio-urop-data/' CONFIDENCE_MAP_FOLDER = 'ILSVRC2012_confidence_maps/' # TEST PARAMS CROP_TYPE = 'proportional' # choose between 'proportional' and 'constant' CONSTANT = 81 # pixel-length of square crop (must be odd) PROPORTION = 0.2 # proportion of height that will be length of square crop IMAGE_ID = int(sys.argv[1]) IMAGE_FILENAME = PATH_TO_DATA + settings.folder_name('img') + settings.get_ind_name(IMAGE_ID) + '.JPEG' # filename of the image including path to poggio-urop-data image_tag = settings.get_ind_name(IMAGE_ID) labels_file = open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json') true_labels = json.load(labels_file) TRUE_CLASS = true_labels[image_tag] # START VGG class vgg16: def __init__(self, imgs, weights=None, sess=None): self.imgs = imgs self.convlayers() self.fc_layers() self.probs = tf.nn.softmax(self.fc3l) if weights is not None and sess is not None: