def multilabel_get_final_activations(url_or_np_array):
image_mean = np.array([107.0, 117.0, 123.0])
input_scale = None
channel_swap = [2, 1, 0]
raw_scale = 255.0
print('loading caffemodel for neurodoll (single class layers)')
start_time = time.time()
if isinstance(url_or_np_array, basestring):
print('infer_one working on url:' + url_or_np_array)
image = url_to_image(url_or_np_array)
elif type(url_or_np_array) == np.ndarray:
image = url_or_np_array
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
# im = Image.open(imagename)
# im = im.resize(required_imagesize,Image.ANTIALIAS)
# in_ = in_.astype(float)
in_ = imutils.resize_keep_aspect(image,
output_size=required_image_size,
output_file=None)
in_ = np.array(in_, dtype=np.float32) #.astype(float)
if len(in_.shape) != 3: #h x w x channels, will be 2 if only h x w
print('got 1-chan image, turning into 3 channel')
#DEBUG THIS , ORDER MAY BE WRONG [what order? what was i thinking???]
in_ = np.array([in_, in_, in_])
elif in_.shape[
2] != 3: #for rgb/bgr, some imgages have 4 chan for alpha i guess
print('got n-chan image, skipping - shape:' + str(in_.shape))
return
# in_ = in_[:,:,::-1] for doing RGB -> BGR
# cv2.imshow('test',in_)
in_ -= np.array((104, 116, 122.0))
in_ = in_.transpose((2, 0, 1))
# shape for input (data blob is N x C x H x W), set data
net.blobs['data'].reshape(1, *in_.shape)
net.blobs['data'].data[...] = in_
# run net and take argmax for prediction
net.forward()
# out = net.blobs['score'].data[0].argmax(axis=0) #for a parse with per-pixel max
out = net.blobs['myfc7'].data[0] #penultimate
print('shape of myfc7:' + str(out.shape))
min = np.min(out)
max = np.max(out)
print('fc7 min {} max {} out shape {}'.format(min, max, out.shape))
out2 = net.blobs['myfc8'].data[0] #final
print('shape of myfc8:' + str(out2.shape))
min = np.min(out2)
max = np.max(out2)
print('fc8 min {} max {} out shape {}'.format(min, max, out2.shape))
cat = np.concatenate((out, out2))
print('shape of cat:' + str(cat.shape))
elapsed_time = time.time() - start_time
print('infer_one elapsed time:' + str(elapsed_time))
return cat
def test_do_resize(self):
curpath = os.path.dirname(imutils.__file__)
parpath = os.path.dirname(curpath)
img_arr = cv2.imread(os.path.join(parpath, 'images/female1.jpg'))
desired_size = (200, 300)
resized = imutils.resize_keep_aspect(img_arr, output_size=desired_size)
actual_size = resized.shape[0:2]
assert (actual_size == desired_size)
print('orig size {} new size {}'.format(img_arr.shape, resized.shape))
def get_final_activation(url_or_np_array,category_index,required_image_size=(256,256)):
#NOTE THIS IS NOT FINISHED , get correct layer if you want to continue - moving to multilabel instead
start_time = time.time()
if isinstance(url_or_np_array, basestring):
print('infer_one working on url:'+url_or_np_array)
image = url_to_image(url_or_np_array)
elif type(url_or_np_array) == np.ndarray:
image = url_or_np_array
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
# im = Image.open(imagename)
# im = im.resize(required_imagesize,Image.ANTIALIAS)
# in_ = in_.astype(float)
in_ = imutils.resize_keep_aspect(image,output_size=required_image_size,output_file=None)
in_ = np.array(in_, dtype=np.float32) #.astype(float)
if len(in_.shape) != 3: #h x w x channels, will be 2 if only h x w
print('got 1-chan image, turning into 3 channel')
#DEBUG THIS , ORDER MAY BE WRONG [what order? what was i thinking???]
in_ = np.array([in_,in_,in_])
elif in_.shape[2] != 3: #for rgb/bgr, some imgages have 4 chan for alpha i guess
print('got n-chan image, skipping - shape:'+str(in_.shape))
return
# in_ = in_[:,:,::-1] for doing RGB -> BGR
# cv2.imshow('test',in_)
in_ -= np.array((104,116,122.0))
in_ = in_.transpose((2,0,1))
# shape for input (data blob is N x C x H x W), set data
net.blobs['data'].reshape(1, *in_.shape)
net.blobs['data'].data[...] = in_
# run net and take argmax for prediction
net.forward()
# out = net.blobs['score'].data[0].argmax(axis=0) #for a parse with per-pixel max
out = net.blobs['siggy'].data[0][category_index] #for the nth class layer #siggy is after sigmoid
min = np.min(out)
max = np.max(out)
print('min {} max {} out shape {}'.format(min,max,out.shape))
out = out*255
min = np.min(out)
max = np.max(out)
print('min {} max {} out after scaling {}'.format(min,max,out.shape))
result = Image.fromarray(out.astype(np.uint8))
# outname = im.strip('.png')[0]+'out.bmp'
# outname = os.path.basename(imagename)
# outname = outname.split('.jpg')[0]+'.bmp'
# outname = os.path.join(out_dir,outname)
# print('outname:'+outname)
# result.save(outname)
# fullout = net.blobs['score'].data[0]
elapsed_time=time.time()-start_time
print('infer_one elapsed time:'+str(elapsed_time))
# cv2.imshow('out',out.astype(np.uint8))
# cv2.waitKey(0)
return out.astype(np.uint8)
def get_single_label_output(url_or_np_array,
required_image_size=(227, 227),
output_layer_name='prob'):
'''
CURRENTLY INOPERATIONAL
:param url_or_np_array:
:param required_image_size:
:param output_layer_name:
:return:
'''
if isinstance(url_or_np_array, basestring):
print('infer_one working on url:' + url_or_np_array)
image = Utils.get_cv2_img_array(url_or_np_array)
elif type(url_or_np_array) == np.ndarray:
image = url_or_np_array
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
# im = Image.open(imagename)
# im = im.resize(required_imagesize,Image.ANTIALIAS)
# in_ = in_.astype(float)
in_ = imutils.resize_keep_aspect(image,
output_size=required_image_size,
output_file=None)
in_ = np.array(in_, dtype=np.float32) #.astype(float)
if len(in_.shape) != 3: #h x w x channels, will be 2 if only h x w
print('got 1-chan image, turning into 3 channel')
#DEBUG THIS , ORDER MAY BE WRONG [what order? what was i thinking???]
in_ = np.array([in_, in_, in_])
elif in_.shape[
2] != 3: #for rgb/bgr, some imgages have 4 chan for alpha i guess
print('got n-chan image, skipping - shape:' + str(in_.shape))
return
# in_ = in_[:,:,::-1] for doing RGB -> BGR : since this is loaded nby cv2 its unecessary
# cv2.imshow('test',in_)
in_ -= np.array((104, 116, 122.0))
in_ = in_.transpose((2, 0, 1))
# shape for input (data blob is N x C x H x W), set data
multilabel_net.blobs['data'].reshape(1, *in_.shape)
multilabel_net.blobs['data'].data[...] = in_
# run net and take argmax for prediction
multilabel_net.forward()
# out = net.blobs['score'].data[0].argmax(axis=0) #for a parse with per-pixel max
out = multilabel_net.blobs[output_layer_name].data[
0] #for the nth class layer #siggy is after sigmoid
min = np.min(out)
max = np.max(out)
print('out {}'.format(out))
def infer_one(url_or_np_array,required_image_size=(256,256)):
start_time = time.time()
if isinstance(url_or_np_array, basestring):
print('infer_one working on url:'+url_or_np_array)
image = url_to_image(url_or_np_array)
elif type(url_or_np_array) == np.ndarray:
image = url_or_np_array
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
# im = Image.open(imagename)
# im = im.resize(required_imagesize,Image.ANTIALIAS)
# in_ = in_.astype(float)
in_ = imutils.resize_keep_aspect(image,output_size=required_image_size,output_file=None)
in_ = np.array(in_, dtype=np.float32) #.astype(float)
if len(in_.shape) != 3:
print('got 1-chan image, turning into 3 channel')
#DEBUG THIS , ORDER MAY BE WRONG
in_ = np.array([in_,in_,in_])
elif in_.shape[2] != 3:
print('got n-chan image, skipping - shape:'+str(in_.shape))
return
# in_ = in_[:,:,::-1] for doing RGB -> BGR
# cv2.imshow('test',in_)
in_ -= np.array((104,116,122.0))
in_ = in_.transpose((2,0,1))
# shape for input (data blob is N x C x H x W), set data
net.blobs['data'].reshape(1, *in_.shape)
net.blobs['data'].data[...] = in_
# run net and take argmax for prediction
net.forward()
out = net.blobs['score'].data[0].argmax(axis=0)
result = Image.fromarray(out.astype(np.uint8))
# outname = im.strip('.png')[0]+'out.bmp'
# outname = os.path.basename(imagename)
# outname = outname.split('.jpg')[0]+'.bmp'
# outname = os.path.join(out_dir,outname)
# print('outname:'+outname)
# result.save(outname)
# fullout = net.blobs['score'].data[0]
elapsed_time=time.time()-start_time
print('infer_one elapsed time:'+str(elapsed_time))
# cv2.imshow('out',out.astype(np.uint8))
# cv2.waitKey(0)
return out.astype(np.uint8)
def get_hydra_output(url_or_image_arr,
out_dir='./',
orig_size=(256, 256),
crop_size=(224, 224),
mean=(104.0, 116.7, 122.7),
gpu=1,
save_data=True,
save_path='/data/jeremy/caffenets/hydra/production/saves',
detection_thresholds=constants.hydra_tg_thresholds,
url=None):
'''
start net, get a bunch of results. DONE: resize to e.g. 250x250 (whatever was done in training) and crop to dims
:param url_or_image_arr_list:#
:param prototxt:
:param caffemodel:
:param out_dir:
:param dims:
:param output_layers:
:param mean:
:return:
'''
detection_thresholds = None
start_time = time.time()
caffe.set_mode_gpu()
caffe.set_device(gpu)
# print('params:'+str(hydra_net.params))
out_layers = hydra_net.outputs
out_layers = put_in_numeric_not_alphabetic_order(out_layers)
# print('out layers: '+str(out_layers))
j = 0
output_names = constants.hydra_tg_heads
# load image, resize, crop, subtract mean, and make dims C x H x W for Caffe
im = Utils.get_cv2_img_array(url_or_image_arr)
if im is None:
logging.warning('could not get image ' + str(url_or_image_arr))
return None
if isinstance(url_or_image_arr, basestring):
print('get_hydra_output working on:' + url_or_image_arr)
print('img size:' + str(im.shape))
im = imutils.resize_keep_aspect(im, output_size=orig_size)
im = imutils.center_crop(im, crop_size)
in_ = np.array(im, dtype=np.float32)
if len(in_.shape) != 3:
print('got 1-chan image, skipping')
return
elif in_.shape[2] != 3:
print('got n-chan image, skipping - shape:' + str(in_.shape))
return
in_ -= mean
in_ = in_.transpose((2, 0, 1)) #W,H,C -> C,W,H
hydra_net.blobs['data'].reshape(1, *in_.shape)
hydra_net.blobs['data'].data[...] = in_
# run net and take argmax for prediction
hydra_net.forward()
out = {}
i = 0
for output_layer in out_layers:
one_out = hydra_net.blobs[output_layer].data[
0] #not sure why the data is nested [1xN] matrix and not a flat [N] vector
second_neuron = copy.copy(
one_out[1]
) #the copy is required - if you dont do it then out gets over-written with each new one_out
second_neuron = round(float(second_neuron), 3)
# print('type:'+str(type(second_neuron)))
name = output_names[i]
if detection_thresholds is None:
out[name] = second_neuron
print('{} for category {} {}'.format(second_neuron, i, name))
elif second_neuron > detection_thresholds[i]:
out[name] = second_neuron
print('{} is past threshold {} for category {} {}'.format(
second_neuron, detection_thresholds[i], i, name))
logging.debug('output for {} {} is {}'.format(output_layer, name,
second_neuron))
# print('final till now:'+str(all_outs)+' '+str(all_outs2))
i = i + 1
logging.debug('all output:' + str(out))
logging.debug('elapsed time:' + str(time.time() - start_time))
if save_data:
if isinstance(url_or_image_arr, basestring):
filename = url_or_image_arr.replace('https://', '').replace(
'http://', '').replace('/', '_')
url = url_or_image_arr
else:
n_chars = 6
filename = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(n_chars)) + '.jpg'
if url is None:
url = 'not_from_url'
Utils.ensure_dir(save_path)
imgname = os.path.join(save_path, filename)
if imgname[:-4] != '.jpg':
imgname = imgname + '.jpg'
cv2.imwrite(imgname, im)
# out['imgname']=filename
out['url'] = url
textfile = os.path.join(save_path, 'output.txt')
with open(textfile, 'a') as fp:
json.dump(out, fp, indent=4)
fp.close()
print('wrote image to {} and output text to {}'.format(
imgname, textfile))
return out
def check_accuracy_deploy(deployproto,
caffemodel,
n_classes,
labelfile,
n_tests=200,
estimate_layer='prob_0',
mean=(110, 110, 110),
scale=None,
finalsize=(224, 224),
resize_size=(250, 250),
gpu=0):
'''
This checks accuracy using the deploy instead of the test net
Its a more definitive test since it checks ifyou are doing the input transforms (resize, mean, scale)
correctly
:param net:
:param n_classes:
:param labelfile:
:param n_tests:
:param estimate_layer:
:param mean:
:param scale:
:param finalsize: img will be cropped to this after resize if any
:param resize_size: resize keeping aspect to this .
in training i have been doing resize to 250x250 and random crop of that to 224x224
:return:
'''
#check accuracy as deployed, using labelfile instead of test net
if gpu == -1:
caffe.set_mode_cpu()
else:
caffe.set_mode_gpu()
caffe.set_device(gpu)
# net = caffe.Net(testproto,caffemodel, caffe.TEST) #apparently this is how test is chosen instead of train if you use a proto that contains both test and train
if caffemodel == '':
caffemodel = None #hack to keep things working, ideally change refs to caffemodel s.t. None is ok
net = caffe.Net(
deployproto, caffe.TEST
) #apparently this is how test is chosen instead of train if you use a proto that contains both test and train
else:
net = caffe.Net(
deployproto, caffe.TEST, weights=caffemodel
) #apparently this is how test is chosen instead of train if you use a proto that contains both test and train
print('checking acc using deploy {} caffemodel {} labels in {} '.format(
deployproto, caffemodel, labelfile))
print('mean {} scale {} gpu {} resize {} finalsize {}'.format(
mean, scale, gpu, resize_size, finalsize))
start_time = time.time()
all_params = [k for k in net.params.keys()]
print('all params:')
print all_params
all_blobs = [k for k in net.blobs.keys()]
print('all blobs:')
print all_blobs
with open(labelfile, 'r') as fp:
lines = fp.readlines()
if not lines:
print('coundlt get lines from file ' + labelfile)
imagelist = [line.split()[0] for line in lines]
labellist = [int(line.split()[1])
for line in lines] #label is a single number for single label
print('1st label {} and file {}, n_classes {} nlabel {} nfile {}'.format(
labellist[0], imagelist[0], n_classes, len(labellist), len(imagelist)))
confmat = np.zeros([n_classes, n_classes])
for t in range(n_tests):
imgfile = imagelist[t]
label = labellist[t]
img_arr = cv2.imread(imgfile)
if img_arr is None:
print('couldnt get ' + imgfile + ' ,skipping')
continue
if resize_size is not None:
img_arr = imutils.resize_keep_aspect(img_arr,
output_size=resize_size)
if finalsize is not None:
img_arr = imutils.center_crop(img_arr, finalsize)
print('in shape ' + str(img_arr.shape))
if mean is not None:
img_arr = img_arr - mean
img_arr = np.transpose(
img_arr, [2, 0, 1]) #hwc->cwh , rgb-bgr not needed (cv2 read)
img_arr = np.array(img_arr, dtype=np.float32)
if scale is not None:
if scale is True:
img_arr = img_arr / 255
else:
img_arr = img_arr / scale
print('img mean {} std {}'.format(np.average(img_arr),
np.std(img_arr)))
#feed img into net
net.blobs['data'].reshape(1, *img_arr.shape)
net.blobs['data'].data[...] = img_arr
net.forward()
est = net.blobs[estimate_layer].data #.data gets the loss
if np.any(np.isnan(est)):
print('got nan in ests, continuing')
continue
best_guess = np.argmax(est)
print('test {}/{}: gt {} est {}->{} '.format(t, n_tests, label, est,
best_guess))
confmat = update_confmat(label, best_guess, confmat)
print(confmat)
print('final confmat')
print(confmat)
elapsed = time.time() - start_time
print('elapsed time {} tpi {} gpu {}'.format(elapsed, elapsed / n_tests,
gpu))
for i in range(n_classes):
p, r, a = precision_recall_accuracy(confmat, i)
print('class {} prec {} rec {} acc {}'.format(i, p, r, a))
return confmat
def get_single_label_output(url_or_np_array,
net,
required_image_size=(224, 224),
resize=(250, 250),
analog_output=True,
from_binary_net=True):
'''
gets the output of a single-label classifier.
:param url_or_np_array:
:param net:
:param required_image_size: the size of the image the net wants (has been trained on), (WxH)
:param resize: resize img to this dimension. if this is > required_image_size then take center crop. pls dont make this < required_image_size
:param analog_output: get the actual class activations not just the max
:param binary_net: get the 2nd neuron output if this is a binary net (forget abt the first one since the neurons would be 'not item/item'
:return:
'''
#the below could be replaced by a call to
if isinstance(url_or_np_array, basestring):
image = url_to_image(url_or_np_array)
elif type(url_or_np_array) == np.ndarray:
image = url_or_np_array
image = Utils.get_cv2_img_array(url_or_np_array)
if image is None:
print('ug didnt manage to get an image...' + str(url_or_np_array))
return
print('multilabel working on image of shape:' + str(image.shape))
# save image to make sure no rgb/bgr funny business
# hash = hashlib.sha1()
# hash.update(str(time.time()))
# print hash.hexdigest()
# name=hash.hexdigest()[:10]+'.jpg'
# print('saving '+name)
# cv2.imwrite(name,image)
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
# im = Image.open(imagename)
# im = im.resize(required_imagesize,Image.ANTIALIAS)
# in_ = in_.astype(float)
if resize:
image = imutils.resize_keep_aspect(image,
output_size=resize,
output_file=None)
#print('original resized to '+str(image.shape))
height, width, channels = image.shape
crop_dx = width - required_image_size[0]
crop_dy = height - required_image_size[1]
if crop_dx != 0:
remove_x_left = crop_dx / 2
remove_x_right = crop_dx - remove_x_left
image = image[:,
remove_x_left:width - remove_x_right, :] #crop center x
#print('removing {} from left and {} from right leaving {}'.format(remove_x_left,remove_x_right,image.shape))
if crop_dy != 0:
remove_y_top = crop_dy / 2
remove_y_bottom = crop_dy - remove_y_top
image = image[remove_y_top:width -
remove_y_bottom, :, :] #crop center y
#print('removing {} from top and {} from bottom leaving {}'.format(remove_x_left,remove_x_right,image.shape))
image = np.array(image, dtype=np.float32) #.astype(float)
if len(image.shape) != 3: #h x w x channels, will be 2 if only h x w
print('got 1-chan image, turning into 3 channel')
#DEBUG THIS , ORDER MAY BE WRONG [what order? what was i thinking???]
image = np.array([image, image, image])
elif image.shape[
2] != 3: #for rgb/bgr, some imgages have 4 chan for alpha i guess
print('got n-chan image, skipping - shape:' + str(image.shape))
return
# image = image[:,:,::-1] for doing RGB -> BGR : since this is loaded nby cv2 its unecessary
# cv2.imshow('test',image)
image -= np.array((104.0, 116.0, 122.0))
image = image.transpose((2, 0, 1))
# shape for input (data blob is N x C x H x W), set data
net.blobs['data'].reshape(1, *image.shape)
net.blobs['data'].data[...] = image
# run net and take argmax for prediction
net.forward()
# out = net.blobs['score'].data[0].argmax(axis=0) #for a parse with per-pixel max
out = net.blobs['prob'].data[
0] #for the nth class layer #siggy is after sigmoid
the_chosen_one = out.argmax()
min = np.min(out)
max = np.max(out)
print('net output: {} answer:class {}'.format(out, the_chosen_one))
if analog_output:
if from_binary_net:
second_neuron_output = out[1]
print('binary output:' + str(out[1]))
return second_neuron_output
return out
return the_chosen_one
