def mask_confidence(image_id,
crop_metric,
model_name,
image_scale,
correctness='top5'):
conf = np.load(
PATH_TO_DATA +
settings.map_filename(settings.CONFIDENCE_MAPTYPE, crop_metric,
model_name, image_scale, image_id))
corr = np.load(PATH_TO_DATA + settings.map_filename(
correctness, crop_metric, model_name, image_scale, image_id))
def get_backprops_by_index(crop_metric, model_name, image_scale, image_id,
indices):
backprops = np.load(
PATH_TO_DATA +
settings.map_filename(settings.BACKPROP_MAPTYPE, crop_metric,
model_name, image_scale, image_id) + '.npz')
for index in indices:
yield backprops[str(list(index))]
def crop_correctness_in_bbx(crop_metric, model_name, image_scale):
'''
Parameters
----------
crop_metric: (float) crop metric for size being used
model_name: (str) model name being used
image_scale: (float) image scale being used
Saves json file titled stats/<crop_metric>/<model_name>/<image_scale>/correct-min-imgs-in-bbx.json. File maps
smalldataset_id to percent error of crops in bbx.
Deals with multiple bbxs by masking and taking average percent correctness across bbxs. Chose to do this, instead of
masking to isolate one big bbx region, because it seems like we're going for percent correct crops per object, so
averaging across bbxs seems more appropriate. TODO confirm
'''
with open(BBX_FILE, 'r') as bbx_file:
all_bbxs = json.load(bbx_file)
intractable_images = settings.get_intractable_images(PATH_TO_DATA, crop_metric, model_name, image_scale)
all_img_pct_correct_in_bbx = {}
for smalldataset_id in range(settings.SMALL_DATASET_SIZE):
top5filename = PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, smalldataset_id) + '.npy'
if not os.path.exists(top5filename):
continue
top5map = np.load(top5filename)
bbx_dims = settings.get_bbx_dims(all_bbxs, smalldataset_id)
pct_correct_in_bbx = 0
crop_size = get_crop_size(smalldataset_id, crop_metric)
for x1, y1, x2, y2 in bbx_dims:
offset = int(crop_size / 2)
_map_height, _map_width = top5map.shape
mx1, my1, mx2, my2 = max(0, x1 - offset), max(0, y1 - offset), min(_map_width, x2 - offset), min(_map_height, y2 - offset)
mx1, my1, mx2, my2 = min(mx1, _map_width), min(my1, _map_height), max(mx2, 0), max(my2, 0) # make sure they didn't go too far the other way
# bbxs that are entirely outside of the map boundary in at least one dimension
if mx1 == mx2:
if mx1 == _map_width: # if we're at the right end...
mx1 -= 2 # adjust left
else:
mx2 += 2 # else adjust right
if my1 == my2:
if my1 == _map_height: # if we're at the bottom end...
my1 -= 2 # adjust up
else:
my2 += 2 # else adjust down
bbx = top5map[my1:my2, mx1:mx2]
pct_correct_in_bbx += np.sum(bbx > 0.) / bbx.size # calculate how much of bbx is classified correctly
pct_correct_in_bbx /= len(bbx_dims) # average percentage - it's all the same type of object
all_img_pct_correct_in_bbx[smalldataset_id] = pct_correct_in_bbx
with open(PATH_TO_OUTPUT_DATA + os.path.join('stats', str(crop_metric), str(model_name), str(image_scale), 'all-img-pct-correct-in-bbx.json'), 'w') as f:
json.dump(all_img_pct_correct_in_bbx, f)
def minimal_image_distribution(num_imgs, crop_metric, model_name, image_scale, strictness):
resize_dim = 150
minimal_image_aggregation = np.zeros((resize_dim, resize_dim))
# img_ids = random.sample(range(100), num_imgs) # for testing on my machine: only a subset of the maps. TODO remove for full job
img_ids = range(3)
for smalldataset_id in img_ids:
# get bbx dimensions
imagenetval_id = settings.convert_id_small_to_imagenetval(smalldataset_id)
image_tag = settings.get_ind_name(imagenetval_id)
with open(BBX_FILE, 'r') as bbx_file:
all_bbxs = json.load(bbx_file)
crop_dims = [bbx[0] for bbx in all_bbxs[image_tag]] # get all x1, y1, x2, y2 crops
minimal_map_f = PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, smalldataset_id)
minimal_map_f = minimal_map_f + '_' + ('l' if strictness == 'loose' else '') + 'map'
minimal_map = np.load(minimal_map_f + '.npy')
image_filename = PATH_TO_DATA + settings.folder_name('img') + image_tag + '.JPEG'
try: # for testing on my machine: if the image is not on my machine, move on. TODO remove for full job
im = Image.open(image_filename)
except OSError:
continue
width, height = im.size
crop_type = 'proportional' if crop_metric <= 1. else 'constant'
crop_size = c_m_p.get_crop_size(height, crop_metric, crop_type) if height <= width else c_m_p.get_crop_size(width, crop_metric, crop_type)
for x1, y1, x2, y2 in crop_dims:
minmap_sub = minimal_map[y1:y2 - crop_size + 1, x1:x2 - crop_size + 1]
minmap_sub = imresize(minmap_sub, (resize_dim, resize_dim))
minimal_image_aggregation += minmap_sub
vis = (minimal_image_aggregation - np.min(minimal_image_aggregation))
vis /= np.max(vis)
vis *= 255.
Image.fromarray(vis).show()
return minimal_image_aggregation
def crop_correctness_across_image(crop_metric, model_name, image_scale):
'''
saves stats/<crop_metric>/<model_name>/<image_scale>/crop-classification-correctness.npy
1x500 vector. Cell i contains smalldataset_id=i's top5map white percentage. If top5 map doesn't exist for this combo, vector gets np.nan.
'''
results = np.zeros(settings.SMALL_DATASET_SIZE)
for smalldataset_id in range(settings.SMALL_DATASET_SIZE):
try:
top5map = np.load(PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, smalldataset_id) + '.npy')
print(smalldataset_id, 'worked')
except FileNotFoundError as e:
print(smalldataset_id, 'failed')
results[smalldataset_id] = np.nan # if there is no map, put in a nan (for nanmean when visualizing) and move to next smalldataset_id)
continue
percent_correct = float(np.sum(top5map > 0.)) / top5map.size
results[smalldataset_id] = percent_correct
print('RESULT:', results)
np.save(PATH_TO_OUTPUT_STATS + os.path.join(str(crop_metric), model_name, str(image_scale), 'crop-classification-correctness.npy'), results)
def backprop_crops(image_id, crop_metric, model_name, image_scale):
'''
note: even with the graph inefficiency, it makes more sense for this function to be one image at a time because the indices are different for each one. This would only change if I figured out a programmatic way to find the right indices.
'''
# Get indices for this map
with open(PATH_TO_DATA + settings.BACKPROP_INDICES_FILENAME,
'r') as indfile:
all_inds = json.load(indfile)
indices = all_inds[str(
(crop_metric, model_name, image_scale, image_id))]
with tf.Session() as sess:
# Set up CNN
model = settings.MODELS[model_name]
batch_size = len(indices)
imgs = tf.placeholder(tf.float32,
[batch_size, model.im_size, model.im_size, 3])
network = model(imgs, sess)
# Create backprop objects
true_labels = json.load(
open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json'))
true_label = true_labels[settings.get_ind_name(image_id)]
y = tf.constant([true_label for __ in range(batch_size)])
err = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=y, logits=network.logits)
dy_dx = tf.gradients(err, network.imgs)
# Scale image, get all crops and run backprop
image = Image.open(PATH_TO_DATA + settings.folder_name('img') +
settings.get_ind_name(image_id) + '.JPEG')
width, height = image.size
image = imresize(image,
(int(width * image_scale), int(height * image_scale)))
crop_type = 'proportional' if crop_metric <= 1. else 'constant'
crop_size = get_crop_size(
height, crop_metric,
crop_type) if height <= width else get_crop_size(
width, crop_metric, crop_type)
all_crops = []
for x1, y1 in indices:
crop = image[y1:y1 + crop_size, x1:x1 + crop_size, :]
# crop = image.crop((x1, y1, x1 + crop_size, y1 + crop_size))
all_crops.append(imresize(crop, (model.im_size, model.im_size)))
# all_crops.append(imresize(image, (model.im_size, model.im_size))) # TODO remove after making sure this code is working
backprops = sess.run(
dy_dx,
feed_dict={network.imgs: model.preprocess(np.array(all_crops))})[0]
# Make backprop results visualizable
backprops = backprops - np.min(backprops, axis=(1, 2), keepdims=True)
backprops = backprops / np.max(backprops, axis=(1, 2), keepdims=True)
backprops = backprops * 255.
backprops = backprops.astype(np.uint8)
# savez results
folder = PATH_TO_DATA + settings.map_folder_name(
settings.BACKPROP_MAPTYPE, crop_metric, model_name, image_scale)
filename = PATH_TO_DATA + settings.map_filename(
settings.BACKPROP_MAPTYPE, crop_metric, model_name, image_scale,
image_id)
if not os.path.exists(folder):
os.makedirs(folder)
np.savez(filename,
**{str(indices[i]): backprops[i]
for i in range(len(indices))})
os.makedirs(folder)
np.savez(filename,
**{str(indices[i]): backprops[i]
for i in range(len(indices))})
def get_backprops_by_index(crop_metric, model_name, image_scale, image_id,
indices):
backprops = np.load(
PATH_TO_DATA +
settings.map_filename(settings.BACKPROP_MAPTYPE, crop_metric,
model_name, image_scale, image_id) + '.npz')
for index in indices:
yield backprops[str(list(index))]
if __name__ == '__main__':
test_inds = [(37, 68), (45, 68), (51, 68), (32, 68), (37, 69), (45, 69),
(51, 69), (32, 69)] # these were for 0.2,resnet,1.0,1
# test_inds = [(242, 54), (237, 54), (247, 54), (242, 49), (237, 49), (247, 49)]
pick_backprop_indices(int(sys.argv[1]), float(sys.argv[2]), sys.argv[3],
float(sys.argv[4]), test_inds)
backprop_crops(int(sys.argv[1]), float(sys.argv[2]), sys.argv[3],
float(sys.argv[4]))
outfile = np.load(PATH_TO_DATA + settings.map_filename(
settings.BACKPROP_MAPTYPE, float(sys.argv[2]), sys.argv[3],
float(sys.argv[4]), sys.argv[1] + '.npz'))
print(outfile.keys())
def get_maxdiff_size_crops(start_id,
end_id,
crop_metric,
model_name,
image_scale,
compare_corr=True):
# For each image id in range(start_id, end_id + 1), finds the crop of size crop_metric and the crop ~2 pixels smaller that are maximally different in confidence. If compare_corr is True, it necessarily finds crops where the smaller one is classified incorrectly and the larger one is classified correctly. This uses the top5 maps for the two scales.
crop_type = 'proportional' if crop_metric <= 1. else 'constant'
folders = [
PATH_TO_DATA + settings.maxdiff_folder_name(
'size', crop_metric, model_name, image_scale,
'diff' if compare_corr else 'any', conf)
for conf in ['high', 'low']
]
for folder in folders:
if not os.path.exists(folder):
os.makedirs(folder)
map_folder = PATH_TO_DATA + settings.maxdiff_folder_name(
'size', crop_metric, model_name, image_scale, 'map')
if not os.path.exists(map_folder):
os.makedirs(map_folder)
# with tf.Session() as sess: # TODO use for testing
# model = settings.MODELS[model_name]
# imgs = tf.placeholder(tf.float32, [None, model.im_size, model.im_size, 3])
# network = model(imgs, sess)
true_labels = json.load(
open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json'))
for image_id in range(start_id, end_id + 1):
image_tag = settings.get_ind_name(image_id)
image_filename = PATH_TO_DATA + settings.folder_name(
'img') + image_tag + '.JPEG'
true_class = true_labels[image_tag]
im = Image.open(image_filename)
if im.mode != 'RGB':
im = im.convert('RGB')
width, height = im.size
im = im.resize((int(width * image_scale), int(height * image_scale)))
width, height = im.size
im = np.asarray(im)
# Get the small crop_metric, crop sizes for the large and small crop_metrics
size_dim = height if height <= width else width
large_size = get_crop_size(
size_dim, crop_metric,
crop_type) if height <= width else get_crop_size(
width, crop_metric, crop_type)
small_metric = 0.194 if crop_metric == 0.2 else 0.394 # TODO change to be a calculation and command
small_size = get_crop_size(size_dim, small_metric, crop_type)
metrics = [crop_metric, small_metric]
# Get the correctness maps (top5, may become a choice between top5 and top1 in the future), and if the call requires diff correctness, check that that's possible
corr_fns = [
PATH_TO_DATA +
settings.map_filename(settings.TOP5_MAPTYPE, metric, model_name,
image_scale, image_id) + '.npy'
for metric in metrics
] # TODO change to allow choice of top1 or top5
lcor, scor = cor_maps = [np.load(corr_fn) for corr_fn in corr_fns]
if compare_corr:
for cor_map in cor_maps:
if not cor_map.any():
print('%s has no correctly classified crops.' % image_tag)
continue
elif cor_map.all():
print('%s has only correctly classified crops.' %
image_tag)
continue
# Get confidence maps
con_fns = [
PATH_TO_DATA +
settings.map_filename(settings.CONFIDENCE_MAPTYPE, metric,
model_name, image_scale, image_id) + '.npy'
for metric in metrics
]
lcon, scon = [np.load(con_fn) for con_fn in con_fns]
# Calculate difference matrices
lrows, lcols = lcon.shape
offset = large_size - small_size # get the metric that bottom and right are off by
tl_sub = scon[:lrows, :
lcols] # for top left, get the top left small crops that correspond to big crops - same shape. It's all of them because the large crops are all adjacent, even if the difference in pixels is >1.
tr_sub = scon[:lrows, offset:lcols +
offset] # for top right, everything that is 'offset' cols over
bl_sub = scon[offset:lrows + offset, :lcols]
br_sub = scon[offset:lrows + offset, offset:lcols + offset]
ctoffset = int(offset / 2)
ct_sub = scon[ctoffset:lrows + ctoffset, ctoffset:lcols + ctoffset]
diffs = {
'tl': lcon -
tl_sub, # use subtraction because we are looking for increase in conf from increase in size
'tr': lcon - tr_sub,
'bl': lcon - bl_sub,
'br': lcon - br_sub,
'ct': lcon - ct_sub
}
# Make map of the largest size change in confidence across all directions of shrinking
change_map = np.maximum.reduce(list(diffs.values()))
np.save(map_folder + str(image_id), change_map)
# Find maxdiff pair by searching for maximally different pairs until one with different correctness is found (if diffcor. Else, this will terminate after one loop as the first pair found will be the maximally different one and therefore the right one for anycor.)
while True:
maxes = {
corner: np.unravel_index(np.argmax(diffs[corner]),
diffs[corner].shape)
for corner in diffs
} # map each corner diff to its argmax (index of maximum confidence diff)
max_dir = max(
[corner for corner in maxes],
key=lambda corner: diffs[corner][tuple(maxes[corner])]
) # get the corner id of the diff whose max change is the highest out of the four max changes
# getting the indices of the maximal confidence increase. Indices are based on the size of the large crop size map. The first index is the index for the large crop, and the second is for the small crop.
corner_max = maxes[max_dir]
if max_dir == 'tl':
lcell, scell = tuple(corner_max), tuple(corner_max)
elif max_dir == 'tr':
lcell, scell = tuple(corner_max), (corner_max[0],
corner_max[1] + offset)
elif max_dir == 'bl':
lcell, scell = tuple(corner_max), (corner_max[0] + offset,
corner_max[1])
elif max_dir == 'br':
lcell, scell = tuple(corner_max), (corner_max[0] + offset,
corner_max[1] + offset)
else:
lcell, scell = tuple(corner_max), (corner_max[0] + ctoffset,
corner_max[1] + ctoffset)
diff_corr = lcor[lcell] != scor[scell]
if diff_corr or not compare_corr:
sy, sx = scell
ly, lx = lcell
lcropped = im[lcell[0]:lcell[0] + large_size,
lcell[1]:lcell[1] + large_size]
scropped = im[scell[0]:scell[0] + small_size,
scell[1]:scell[1] + small_size]
# lcropped = imresize(lcropped, (network.im_size, network.im_size))
# scropped = imresize(scropped, (network.im_size, network.im_size))
# result = sess.run(network.probs, feed_dict={network.imgs: np.array([lcropped, scropped])}) # run and see if it works later. Without this, the sess isn't actually being used - this is for internal test.
break
else: # if that location wasn't diffcorr, set the diff's entry to -2.
diffs[max_dir][lcell] = -2.
lfolder, sfolder = folders
np.save(lfolder + str(image_id), lcropped)
np.save(sfolder + str(image_id), scropped)
def create_confidence_map(start_id,
end_id,
crop_metric,
model_name,
image_scale,
make_cmap=True,
make_top5=True,
make_top1=True,
make_distance=True):
# Get the crop type - if the crop_metric is a fraction, it's a proportion. If it's larger than 1, it's a constant crop size.
crop_type = 'proportional' if crop_metric <= 1. else 'constant'
# Prepare folders
maptypes = [
settings.CONFIDENCE_MAPTYPE, settings.TOP5_MAPTYPE,
settings.TOP1_MAPTYPE, settings.DISTANCE_MAPTYPE
]
folders = [
PATH_TO_DATA +
settings.map_folder_name(maptype, crop_metric, model_name, image_scale)
for maptype in maptypes
]
for folder in folders:
if not os.path.exists(folder):
os.makedirs(folder)
# Get the right model
model = settings.MODELS[model_name]
# Before anything else happens, so we only set up once despite multiple images, make a network for each GPU
config = tf.ConfigProto(log_device_placement=True,
allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
networks = []
for i in range(NUM_GPUS):
gpu = 'device:GPU:%d' % i
with tf.device(gpu):
imgs = tf.placeholder(
tf.float32, [BATCH_SIZE, model.im_size, model.im_size, 3])
if not model_name == 'alexnet':
network = model(imgs, sess, reuse=None if i == 0 else True)
else:
network = model(imgs, sess)
networks.append(network)
# Get each map called for!
labels_file = open('caffe_ilsvrc12/' + settings.DATASET + '-labels.json')
true_labels = json.load(labels_file)
image_ids = range(start_id, end_id + 1)
for image_id in image_ids:
f = open('small-dataset-to-imagenet.txt')
lines = f.readlines()
image_tag = lines[image_id].split(" ", 1)[0]
print(image_tag)
#image_tag = settings.get_ind_name(image_id)
image_filename = PATH_TO_DATA + settings.folder_name(
'img') + image_tag #+ ('.png' if image_id == 50001 else'.JPEG')
true_class = true_labels[image_tag[:-5]] if image_id != 50001 else 266
im = Image.open(image_filename)
if im.mode != 'RGB':
im = im.convert(
'RGB'
) # make sure bw images are 3-channel, because they get opened in L mode
width, height = im.size
# Resize image based on image_scale
im = im.resize((int(width * image_scale), int(height * image_scale)))
width, height = im.size
# TODO do resizing experiment im = imresize(im, (width*image_scale, height*image_scale)) # resize image as needed
crop_size = get_crop_size(
height, crop_metric,
crop_type) if height <= width else get_crop_size(
width, crop_metric, crop_type)
crop_size -= 2 # reduce size by two pixels for small crops
crop_x1s = range(width - crop_size + 1)
crop_y1s = range(height - crop_size + 1)
crop_dims = itertools.product(crop_x1s, crop_y1s)
C, O, F, D = (np.zeros((height - crop_size + 1, width - crop_size + 1))
for __ in range(4))
crop_dims = list(crop_dims)
total_crops = len(crop_dims)
crop_index = 0
overall_start_time = time.clock()
print('TOTAL CROPS:', total_crops)
sys.stdout.flush()
while crop_index < total_crops: # While we haven't exhausted crops TODO see if the logic needs to be changed
all_cropped_imgs = []
stub_crops = 0 # Initializing to 0 in case there's an image with no stub crops (i.e. number of crops is a multiple of 64)
map_indices = [
] # The indices that these confidences will be mapped to
for i in range(
BATCH_SIZE * NUM_GPUS
): # Get BATCH_SIZE crops for each GPU (total of NUM_GPUS)
if crop_index == total_crops: # If we're on the last round and it's not all 64, repeat the last crop for the rest
stub_crops = (
BATCH_SIZE * NUM_GPUS
) - i # the number of crops still needed, because i in this round is the number of crops that have already been filled in
cropped = imresize(
cropped, (model.im_size, model.im_size)
) # resize the last one (from previous round) permanently. There will be a previous one due to the while logic
for i in range(stub_crops):
all_cropped_imgs.append(
cropped
) # fill in the rest of the slots with the resized last crop
break
# If not on the last one, continue on to fill in the next crop
x1, y1 = crop_dims[crop_index]
map_indices.append((x1, y1))
cropped = im.crop((x1, y1, x1 + crop_size, y1 + crop_size))
all_cropped_imgs.append(
imresize(cropped, (model.im_size, model.im_size)))
crop_index += 1 # Increment to get the next crop dimensions
start_time = time.clock()
network_probs = list(map(lambda x: x.probs, networks))
num_crops = len(all_cropped_imgs)
partition_cropped_imgs = [
all_cropped_imgs[int(i * num_crops / NUM_GPUS):min(
num_crops, int((i + 1) * num_crops / NUM_GPUS))]
for i in range(NUM_GPUS)
] # TODO yikes is this correct
prob = np.array(
sess.run(network_probs,
feed_dict={
networks[i].imgs: model.preprocess(
np.array(partition_cropped_imgs[i]))
for i in range(NUM_GPUS)
}))
# prob = sess.run(vgg.probs, feed_dict={vgg.imgs: all_cropped_imgs})
end_time = time.clock()
print('Time for running one size-' + str(BATCH_SIZE), 'batch:',
end_time - start_time, 'seconds')
print('CROPS COMPLETED SO FAR:', crop_index)
sys.stdout.flush()
# plot the confidences in the map. For final iteration, which likely has <BATCH_SIZE meaningful crops, the index list being of shorter length will cause them to be thrown.
confidence = prob[:, :, true_class].reshape(BATCH_SIZE * NUM_GPUS)
for i in range(len(map_indices)):
c, r = map_indices[i]
C[r, c] = confidence[i]
# plot the top-5 and top-1 binary correctness maps
flat_probs = prob.reshape(
(NUM_GPUS * BATCH_SIZE, settings.NUM_CLASSES))
sorted_classes = flat_probs.argsort(axis=1)
top_5 = sorted_classes[:, -5:]
top_1 = sorted_classes[:, -1].squeeze()
for i in range(len(map_indices)):
c, r = map_indices[i]
O[r, c] = 255. if top_1[i] == true_class else 0.
F[r, c] = 255. if true_class in top_5[i] else 0.
# plot the distance map
sixth = sorted_classes[:, 6]
for i in range(len(map_indices)):
c, r = map_indices[i]
if true_class in top_5[i]:
D[r, c] = 255. * flat_probs[i][true_class] - flat_probs[i][
sixth[i]]
else:
D[r, c] = 0.
overall_end_time = time.clock()
print(
'Time for overall cropping and map-building process with size-' +
str(BATCH_SIZE), 'batch:', overall_end_time - overall_start_time,
'seconds')
sys.stdout.flush()
# Make confidence map, save to confidence map folder
if make_cmap:
np.save(
PATH_TO_DATA +
settings.map_filename(settings.CONFIDENCE_MAPTYPE, crop_metric,
model_name, image_scale, image_id) +
'_small', C)
# Save binary correctness maps
if make_top5:
np.save(
PATH_TO_DATA +
settings.map_filename(settings.TOP5_MAPTYPE, crop_metric,
model_name, image_scale, image_id) +
'_small', F)
if make_top1:
np.save(
PATH_TO_DATA +
settings.map_filename(settings.TOP1_MAPTYPE, crop_metric,
model_name, image_scale, image_id) +
'_small', O)
# Save distance maps
if make_distance:
np.save(
PATH_TO_DATA +
settings.map_filename(settings.DISTANCE_MAPTYPE, crop_metric,
model_name, image_scale, image_id) +
'_small', D)
print(image_tag)
def create_human_size_minimal_image_maps(image_id, crop_metric, model_name,
image_scale, loose):
PATH_TO_DATA = '/om/user/sanjanas/min-img-data/human-image-comparison/'
fname = PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
image_id) + '.npy'
fname = PATH_TO_DATA + 'top5/0.4/resnet/1.0/' + str(image_id) + '.npy'
if not os.path.isfile(fname):
print(fname)
print('large map not found')
return -1, -1
l_top5 = np.load(fname)
fname = PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
image_id) + '_small.npy'
fname = PATH_TO_DATA + 'top5/0.4/resnet/1.0/' + str(
image_id) + '_small.npy'
if not os.path.isfile(fname):
print('small map not found')
return -1, -1
s_top5 = np.load(fname)
r, c = l_top5.shape
M = np.zeros((r, c))
print('began making map')
for i in range(r):
for j in range(c):
self = l_top5[i, j]
window = s_top5[i:i + 3, j:j +
3] # get all the possible shrinks for this crop
if loose:
if self: # if the current crop is correctly classified...
if not np.all(
window
): # if any cell in the window is incorrectly classified...
M[i,
j] = 1. # ...the current crop is a positive minimal image. Otherwise, it's not minimal.
else: # if the current crop is incorrectly classified...
if np.any(
window
): # if any cell in the window is correctly classified...
M[i,
j] = 1. # ...the current crop is a negative minimal image. Otherwise, it's not minimal.
else: # we are looking for strict minimal image maps
if self: # if the current crop is correctly classified...
if not np.any(
window
): # if all crops in the window are incorrectly classified...
M[i,
j] = 1. # ...the current crop is a positive minimal image. Otherwise, it's not minimal.
else: # if the current crop is incorrectly classified...
if np.all(
window
): # if all the crops in the window are correctly classified...
M[i,
j] = 1. # ...the current crop is a negative minimal image. Otherwise, it's not minimal.
print('finished making map')
# save map
if loose:
print('saving loose map')
np.save(
PATH_TO_DATA + 'top5/0.4/resnet/1.0/' + str(image_id) +
'_small_lmap.npy', M)
# np.save(PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_small_lmap.npy', M)
else:
print('saving strict map')
np.save(
PATH_TO_DATA + 'top5/0.4/resnet/1.0/' + str(image_id) +
'_small_map.npy', M)
# np.save(PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_small_map.npy', M)
# calculate map statistics
num_pos_min_imgs = (M > 0.).sum()
num_neg_min_imgs = (M < 0.).sum()
return num_pos_min_imgs / float(M.size), num_neg_min_imgs / float(M.size)
def create_size_minimal_image_maps(image_id, crop_metric, model_name,
image_scale, loose):
fname = PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
image_id) + '.npy'
if not os.path.isfile(fname):
return -1, -1
l_top5 = np.load(fname)
fname = PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
image_id) + '_small.npy'
if not os.path.isfile(fname):
return -1, -1
s_top5 = np.load(fname)
r, c = l_top5.shape
M = np.zeros((r, c))
for i in range(r):
for j in range(c):
self = l_top5[i, j]
window = s_top5[i:i + 3, j:j +
3] # get all the possible shrinks for this crop
if loose:
if self: # if the current crop is correctly classified...
if not np.all(
window
): # if any cell in the window is incorrectly classified...
M[i,
j] = 1. # ...the current crop is a positive minimal image. Otherwise, it's not minimal.
else: # if the current crop is incorrectly classified...
if np.any(
window
): # if any cell in the window is correctly classified...
M[i,
j] = -1. # ...the current crop is a negative minimal image. Otherwise, it's not minimal.
else: # we are looking for strict minimal image maps
if self: # if the current crop is correctly classified...
if not np.any(
window
): # if all crops in the window are incorrectly classified...
M[i,
j] = 1. # ...the current crop is a positive minimal image. Otherwise, it's not minimal.
else: # if the current crop is incorrectly classified...
if np.all(
window
): # if all the crops in the window are correctly classified...
M[i,
j] = -1. # ...the current crop is a negative minimal image. Otherwise, it's not minimal.
# save map
if loose:
np.save(
PATH_TO_DATA +
settings.map_filename(settings.TOP5_MAPTYPE, crop_metric,
model_name, image_scale, image_id) +
'_small_lmap.npy', M)
else:
np.save(
PATH_TO_DATA +
settings.map_filename(settings.TOP5_MAPTYPE, crop_metric,
model_name, image_scale, image_id) +
'_small_map.npy', M)
# calculate map statistics
num_pos_min_imgs = (M > 0.).sum()
num_neg_min_imgs = (M < 0.).sum()
return num_pos_min_imgs / float(M.size), num_neg_min_imgs / float(M.size)
def create_location_minimal_image_maps(image_id, crop_metric, model_name, image_scale, loose, k=1):
'''
image_id (int): the small dataset id of the image we are finding minimal images for
crop_metric (float): the crop metric we are referencing
model_name (string): the model that we are referencing
image_scale (float): the image scale we are referencing
loose (bool): loose minimal images if True else strict minimal images
k (int): the square size that we are looking for minimal image change within; should be even
'''
fname = PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '.npy'
if not os.path.isfile(fname):
return -1, -1
top5map = np.load(fname)
r, c = top5map.shape
M = np.zeros((r, c))
for i in range(r):
for j in range(c):
offset = int(k / 2)
self = top5map[i, j]
# make minimal image map
if loose:
window = top5map[max(0, i - offset):min(r-1, i + offset)+1, max(0, j - offset):min(c-1, j + offset)+1] # get the k-side-length window centered at current cell
if self: # if the current cell is nonzero...
if not np.all(window): # ...and if any part of the window is zero...
M[i, j] = 1. # ...this is a positive minimal image. If no other part of the window is zero, i.e. everything is nonzero, this is not a minimal image.
else: # if the current cell is zero...
if np.any(window): # ...and if any part of the window is nonzero...
M[i, j] = -1. # ...this is a negative minimal image. If no other part of the window is nonzero, i.e. everything is zero, this is not a minimal image.
else: # we are looking for strict minimal images
if self: # if the current cell is nonzero...
top5map[i, j] = 0. # temporarily set the current cell to zero
window = top5map[max(0, i - offset):min(r-1, i + offset)+1, max(0, j - offset):min(c-1, j + offset)+1] # get the k-side-length window centered at current cell
if not np.any(window): # ...and if no part of the window is nonzero...
M[i, j] = 1. # ...this is a positive minimal image. If some part of the window is nonzero, i.e. a surrounding pixel is nonzero, this is not a minimal image.
top5map[i, j] = self # reset current cell
else: # if the current cell is zero...
top5map[i, j] = 255. # temporarily set the current cell to nonzero
window = top5map[max(0, i - offset):min(r-1, i + offset)+1, max(0, j - offset):min(c-1, j + offset)+1] # get the k-side-length window centered at current cell
if np.all(window): # ...and if the entire window is nonzero...
M[i, j] = -1. # ...this is a negative minimal image. If some part of the window is zero, i.e. a surrounding pixel is zero, this is not a minimal image.
top5map[i, j] = self # reset current cell
# save map
if loose:
np.save(PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_lmap.npy', M)
else:
np.save(PATH_TO_DATA + settings.map_filename(settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale, image_id) + '_map.npy', M)
# calculate map statistics
num_pos_min_imgs = (M > 0.).sum()
num_neg_min_imgs = (M < 0.).sum()
return num_pos_min_imgs/float(M.size), num_neg_min_imgs/float(M.size)
def get_human_maxdiff_coordinates(crop_metric,
model_name,
image_scale,
compare_corr=True):
# for all human images, return coordinates of maximally different crops (shift)
PATH_TO_DATA = '/om/user/sanjanas/min-img-data/human-image-comparison/'
crop_type = 'proportional' if crop_metric <= 1. else 'constant'
maxdiff_coordinates = {}
for image_id in range(50002, 50009):
correctness_filename = PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
image_id) + '.npy'
try:
cor_map = np.load(correctness_filename)
except FileNotFoundError:
continue
if compare_corr:
if not cor_map.any():
print('%s has no correctly classified crops.' % image_id)
continue
elif cor_map.all():
print('%s has only correctly classified crops.' % image_id)
continue
con_map_filename = PATH_TO_DATA + settings.map_filename(
settings.CONFIDENCE_MAPTYPE, crop_metric, model_name, image_scale,
image_id) + '.npy'
con_map = np.load(con_map_filename)
down_diff = np.diff(
con_map, axis=0) # apparently assumes step_size=1 (adjacency)
up_diff = -1. * down_diff
right_diff = np.diff(con_map)
left_diff = -1. * right_diff
diffs = {
'up': up_diff,
'down': down_diff,
'left': left_diff,
'right': right_diff
}
while True:
maxes = {
direction: np.unravel_index(np.argmax(diffs[direction]),
diffs[direction].shape)
for direction in diffs
} # map each directional diff to its argmax (index of its maximum confidence diff)
max_dir = max(
[direction for direction in maxes],
key=lambda direction: diffs[direction][tuple(maxes[direction])]
) # get the direction of the diff whose max confidence is the highest out of the four max confidences
# depending on the max-confidence direction, get the argmax of that direction. The more confident crop will be offset by 1 in a way that depends on the direction.
if max_dir == 'up':
up_max = maxes['up']
gcell, cell = (
tuple(up_max), (up_max[0] + 1, up_max[1])
) # up (and left) are like this because when up and left diffs are made, the negation also changes the direction in which your step goes. it goes down -> up; right -> left.
elif max_dir == 'down':
down_max = maxes['down']
cell, gcell = (tuple(down_max), (down_max[0] + 1, down_max[1]))
elif max_dir == 'left':
left_max = maxes['left']
gcell, cell = (tuple(left_max), (left_max[0], left_max[1] + 1))
else:
right_max = maxes['right']
cell, gcell = (tuple(right_max), (right_max[0],
right_max[1] + 1))
diff_correctness = cor_map[cell] != cor_map[gcell]
if diff_correctness or not compare_corr:
break
else:
if max_dir in ['up', 'left']:
diffs[max_dir][
gcell] = -2. # for the diff where that was the argmax, mark the cell containing it to something lower than any real entry (-1. <= real entry <= 1.) This is gcell for up, left and cell for down, right because the lower-indexed cell is always the one that contained the confidence originally
else:
diffs[max_dir][cell] = -2.
maxdiff_coordinates[image_id] = (gcell, cell,
con_map[gcell] - con_map[cell])
return maxdiff_coordinates
def get_maxdiff_coordinates(start_id,
end_id,
crop_metric,
model_name,
image_scale,
axis,
compare_corr=True):
# for all smalldataset images start_id through end_id, return coordinates of maximally different crops
print('ENTERED COORDS FUNCTION')
crop_type = 'proportional' if crop_metric <= 1. else 'constant'
maxdiff_coordinates = {}
print('ENTERING FOR LOOP')
for smalldataset_id in range(start_id, end_id + 1):
print('CURRENTLY:', smalldataset_id)
if axis == 'scale': # scale (size)-based minimal images
sfxs = ['', '_small']
large_size = get_crop_size(smalldataset_id, crop_metric)
small_size = large_size - 2
# Get correctness maps and check if diffcor possible
corr_fns = (PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
smalldataset_id) + sfx + '.npy' for sfx in sfxs)
try:
lcor, scor = cor_maps = [
np.load(corr_fn) for corr_fn in corr_fns
]
except FileNotFoundError:
continue
if compare_corr:
if not lcor.any():
print('%s has no correctly classified crops. SCALE' %
smalldataset_id)
continue
elif scor.all():
print('%s has only correctly classified crops.' %
smalldataset_id)
continue
# Get confidence maps
con_fns = (PATH_TO_DATA + settings.map_filename(
settings.CONFIDENCE_MAPTYPE, crop_metric, model_name,
image_scale, smalldataset_id) + sfx + '.npy' for sfx in sfxs)
lcon, scon = [np.load(con_fn) for con_fn in con_fns]
# Calculate difference matrices
lrows, lcols = lcon.shape
offset = large_size - small_size # get the metric that bottom and right are off by
tl_sub = scon[:lrows, :
lcols] # for top left, get the top left small crops that correspond to big crops - same shape. It's all of them because the large crops are all adjacent, even if the difference in pixels is >1.
tr_sub = scon[:lrows, offset:lcols +
offset] # for top right, everything that is 'offset' cols over
bl_sub = scon[offset:lrows + offset, :lcols]
br_sub = scon[offset:lrows + offset, offset:lcols + offset]
ctoffset = int(offset / 2)
ct_sub = scon[ctoffset:lrows + ctoffset, ctoffset:lcols + ctoffset]
diffs = {
'tl': lcon -
tl_sub, # use subtraction because we are looking for increase in conf from increase in size
'tr': lcon - tr_sub,
'bl': lcon - bl_sub,
'br': lcon - br_sub,
'ct': lcon - ct_sub
}
# Find maxdiff pair by searching for maximally different pairs until one with different correctness is found (if diffcor. Else, this will terminate after one loop as the first pair found will be the maximally different one and therefore the right one for anycor.)
while True:
# print('ENTERED WHILE LOOP')
maxes = {
corner: np.unravel_index(np.argmax(diffs[corner]),
diffs[corner].shape)
for corner in diffs
} # map each corner diff to its argmax (index of maximum confidence diff)
max_dir = max(
[corner for corner in maxes],
key=lambda corner: diffs[corner][tuple(maxes[corner])]
) # get the corner id of the diff whose max change is the highest out of the four max changes
# getting the indices of the maximal confidence increase. Indices are based on the size of the large crop size map. The first index is the index for the large crop, and the second is for the small crop.
corner_max = maxes[max_dir]
if max_dir == 'tl':
lcell, scell = tuple(corner_max), tuple(corner_max)
elif max_dir == 'tr':
lcell, scell = tuple(corner_max), (corner_max[0],
corner_max[1] + offset)
elif max_dir == 'bl':
lcell, scell = tuple(corner_max), (corner_max[0] + offset,
corner_max[1])
elif max_dir == 'br':
lcell, scell = tuple(corner_max), (corner_max[0] + offset,
corner_max[1] + offset)
else:
lcell, scell = tuple(corner_max), (corner_max[0] +
ctoffset,
corner_max[1] +
ctoffset)
diff_corr = lcor[lcell] != scor[scell]
if diff_corr or not compare_corr:
break
else: # if that location wasn't diffcorr, set the diff's entry to -2.
diffs[max_dir][lcell] = -2.
maxdiff_coordinates[smalldataset_id] = (lcell, scell,
lcon[lcell] - scon[scell])
else: # shift-based maxdiff
correctness_filename = PATH_TO_DATA + settings.map_filename(
settings.TOP5_MAPTYPE, crop_metric, model_name, image_scale,
smalldataset_id) + '.npy'
try:
cor_map = np.load(correctness_filename)
except FileNotFoundError:
continue
if compare_corr:
if not cor_map.any():
print('%s has no correctly classified crops.' %
smalldataset_id)
continue
elif cor_map.all():
print('%s has only correctly classified crops.' %
smalldataset_id)
continue
con_map_filename = PATH_TO_DATA + settings.map_filename(
settings.CONFIDENCE_MAPTYPE, crop_metric, model_name,
image_scale, smalldataset_id) + '.npy'
con_map = np.load(con_map_filename)
down_diff = np.diff(
con_map, axis=0) # apparently assumes step_size=1 (adjacency)
up_diff = -1. * down_diff
right_diff = np.diff(con_map)
left_diff = -1. * right_diff
diffs = {
'up': up_diff,
'down': down_diff,
'left': left_diff,
'right': right_diff
}
while True:
# print('ENTERED WHILE LOOP')
maxes = {
direction: np.unravel_index(np.argmax(diffs[direction]),
diffs[direction].shape)
for direction in diffs
} # map each directional diff to its argmax (index of its maximum confidence diff)
max_dir = max(
[direction for direction in maxes],
key=lambda direction: diffs[direction][tuple(maxes[
direction])]
) # get the direction of the diff whose max confidence is the highest out of the four max confidences
# depending on the max-confidence direction, get the argmax of that direction. The more confident crop will be offset by 1 in a way that depends on the direction.
if max_dir == 'up':
up_max = maxes['up']
gcell, cell = (
tuple(up_max), (up_max[0] + 1, up_max[1])
) # up (and left) are like this because when up and left diffs are made, the negation also changes the direction in which your step goes. it goes down -> up; right -> left.
elif max_dir == 'down':
down_max = maxes['down']
cell, gcell = (tuple(down_max), (down_max[0] + 1,
down_max[1]))
elif max_dir == 'left':
left_max = maxes['left']
gcell, cell = (tuple(left_max), (left_max[0],
left_max[1] + 1))
else:
right_max = maxes['right']
cell, gcell = (tuple(right_max), (right_max[0],
right_max[1] + 1))
diff_correctness = cor_map[cell] != cor_map[gcell]
if diff_correctness or not compare_corr:
break
else:
if max_dir in ['up', 'left']:
diffs[max_dir][
gcell] = -2. # for the diff where that was the argmax, mark the cell containing it to something lower than any real entry (-1. <= real entry <= 1.) This is gcell for up, left and cell for down, right because the lower-indexed cell is always the one that contained the confidence originally
else:
diffs[max_dir][cell] = -2.
maxdiff_coordinates[smalldataset_id] = (gcell, cell,
con_map[gcell] -
con_map[cell])
print('COORDS COMPLETE')
return maxdiff_coordinates
