def meanShift(hsv, f1, f2, rois, mask):
if f1 is not None:
term_crit = ( cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 1 )
#import pdb; pdb.set_trace()
ff1 = cv2.normalize(f1,alpha=0,beta=255,norm_type=cv2.NORM_MINMAX)
ff2 = cv2.normalize(f2,alpha=0,beta=255,norm_type=cv2.NORM_MINMAX)
dst1 = cv2.calcBackProject([hsv], [0,2], ff1, [70, 180, 10, 255], 1)
dst2 = cv2.calcBackProject([hsv], [0,2], ff2, [70, 180, 10, 255], 1)
bMask = cv2.inRange(hsv, np.array((70., 0., 10.)), np.array((255.,255.,50.)))
wMask = cv2.inRange(hsv, np.array((70., 0., 200.)), np.array((255.,255.,255.)))
dst1 = cv2.bitwise_and(dst1, mask)
dst2 = cv2.bitwise_and(dst2, mask)
dst1 = cv2.bitwise_and(dst1, wMask)
dst2 = cv2.bitwise_and(dst2, bMask)
dstVis1 = cv2.applyColorMap(dst1, cv2.COLORMAP_JET)
dstVis2 = cv2.applyColorMap(dst2, cv2.COLORMAP_JET)
nRois = []
for (x0,y0), (x1,y1), flag in rois:
if flag:
x,y,w,h = cv2.meanShift(dst1, (x0,y0,x1-x0,y1-y0), term_crit)[1]
nRois.append( [(x, y), (x+w, y+h), flag] )
dst1[y:y+h, x:x+w] = 0
else:
x,y,w,h = cv2.meanShift(dst2, (x0,y0,x1-x0,y1-y0), term_crit)[1]
nRois.append( [(x, y), (x+w, y+h), flag] )
dst2[y:y+h, x:x+w] = 0
#cv2.imshow('dst1', dstVis1)
#cv2.imshow('dst2', dstVis2)
return nRois
return []
def draw_all(imageForNet, heatmaps, currIndex, div=4., norm=False):
netDecreaseFactor = float(imageForNet.shape[0]) / float(heatmaps.shape[2]) # 8
resized_heatmaps = np.zeros(shape=(heatmaps.shape[0], heatmaps.shape[1], imageForNet.shape[0], imageForNet.shape[1]))
num_maps = heatmaps.shape[1]
combined = None
for i in range(0, num_maps):
heatmap = heatmaps[0,i,:,:]
resizedHeatmap = cv2.resize(heatmap, (0,0), fx=netDecreaseFactor, fy=netDecreaseFactor)
minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(resizedHeatmap)
if i==currIndex and currIndex >=0:
resizedHeatmap = np.abs(resizedHeatmap)
resizedHeatmap = (resizedHeatmap*255.).astype(dtype='uint8')
im_color = cv2.applyColorMap(resizedHeatmap, cv2.COLORMAP_JET)
resizedHeatmap = cv2.addWeighted(imageForNet, 1, im_color, 0.3, 0)
cv2.circle(resizedHeatmap, (int(maxLoc[0]),int(maxLoc[1])), 5, (255,0,0), -1)
return resizedHeatmap
else:
resizedHeatmap = np.abs(resizedHeatmap)
if combined is None:
combined = np.copy(resizedHeatmap);
else:
if i <= num_maps-2:
combined += resizedHeatmap;
if norm:
combined = np.maximum(0, np.minimum(1, combined));
if currIndex < 0:
combined /= div
combined = (combined*255.).astype(dtype='uint8')
im_color = cv2.applyColorMap(combined, cv2.COLORMAP_JET)
combined = cv2.addWeighted(imageForNet, 0.5, im_color, 0.5, 0)
cv2.circle(combined, (int(maxLoc[0]),int(maxLoc[1])), 5, (255,0,0), -1)
return combined
def read_and_generate_heatmap(input_path, output_path):
original_img = cv2.imread(input_path).astype(np.float32)
width, height, _ = original_img.shape
im = process_image(cv2.resize(original_img,(224,224)))
out = model.predict(im)
class_weights = model.layers[-1].get_weights()[0]
print("predictions", out[0])
conv_output = out[2][0,:,:,:]
#Create the class activation map.
cam = np.zeros(dtype = np.float32, shape = conv_output.shape[1:3])
class_to_visualize = 1 # 0 for bad, 1 for good
for i, w in enumerate(class_weights[:, class_to_visualize]):
cam += w * conv_output[i, :, :]
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
heatmap[np.where(cam < 0.2)] = 0
temp = heatmap*0.5 + original_img
file_name = input_path.split("/")[-1].split(".jpg")[0]
cv2.imwrite('output/' + input_path.split("/")[-1],original_img)
cv2.imwrite('output/{}_aesthetics.jpg'.format(file_name), temp)
top_3_hits = np.argsort(out[1], axis=1)[0][::-1][:3]
print("predictions", top_3_hits)
semantic_tags = [ semantics.ix[hit + 1].semantic[1:] for hit in top_3_hits] ##NOTE: Quick fix to remove space
conv_output = out[3][0,:,:,:]
#Create the class activation map.
nth_top_semantic = 0
for nth_top_semantic in range(len(top_3_hits)):
cam = np.zeros(dtype = np.float32, shape = conv_output.shape[1:3])
class_to_visualize = top_3_hits[nth_top_semantic] # 0 for bad, 1 for good
for i, w in enumerate(class_weights[:, class_to_visualize]):
cam += w * conv_output[i, :, :]
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
heatmap[np.where(cam < 0.2)] = 0
temp = heatmap*0.5 + original_img
file_name = input_path.split("/")[-1].split(".jpg")[0]
cv2.imwrite('output/{}_{}.jpg'.format(file_name, semantic_tags[nth_top_semantic]), temp)
def showImgBlend(img1, img2, coef=(0.5, 0.5), title='image'):
"""
Displays two images blended. Second image is blended with color map
:param img1: image (2D nparray)
:param img2: image (2D nparray). This image will be blended with colormap
:param coef: tuple with blending coefficients
:param title: title of window
"""
rows1, cols1 = img1.shape
rows2, cols2 = img2.shape
size = (np.max(rows1, rows2), np.max(cols1, cols2))
img1_filled = imgp.fillImg(img1, size)
img2_filled = imgp.fillImg(img2, size)
img_show1 = ((img1_filled.astype(np.float) - np.min(img1_filled)) / np.max(img1_filled)) * 255
img_show2 = ((img2_filled.astype(np.float) - np.min(img2_filled)) / np.max(img2_filled)) * 255
img1_bgr = cv2.cvtColor(np.uint8(img_show1), cv2.COLOR_GRAY2BGR)
img2_bgr = cv2.cvtColor(np.uint8(img_show2), cv2.COLOR_GRAY2BGR)
img2_bgr = cv2.applyColorMap(img2_bgr, cv2.COLORMAP_HOT)
dst = cv2.addWeighted(img1_bgr, coef[0], img2_bgr, coef[1], 0)
cv2.imshow(title, dst)
cv2.waitKey(0)
cv2.destroyAllWindows()
def face_detection(cpath, dev=0):
cap = cv.VideoCapture(dev)
face_cascade = cv.CascadeClassifier(cpath + 'haarcascade_frontalface_default.xml')
f = None
color = None
while(True):
key = cv.waitKey(1) & 0xFF
ret, frame = cap.read()
frame_gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
if key in [ord(x) for x in stage_index]:
color = color_maps[stage_index.index(chr(key))]
if key == 27:
break
if not key == 255 and key-48 < len(filter_dict):
f = filter_dict[key-48]
faces = face_cascade.detectMultiScale(frame_gray, 1.2, 3)
for (x, y, w, h) in faces:
roi = frame[y:y+h, x:x+w]
# we can only apply the filter if the roi exists
if roi is not None and f is not None:
frame[y:y+h, x:x+w] = f(frame[y:y+h, x:x+w])
# to apply a colorMap to the roi
if color is not None:
frame[y:y+h, x:x+w] = cv.applyColorMap(frame[y:y+h, x:x+w], color)
cv.imshow('face_detector', frame)
cv.destroyAllWindows()
def draw_predicted_heatmap(heatmap, input_size):
heatmap_resized = cv2.resize(heatmap, (input_size, input_size))
output_img = None
tmp_concat_img = None
h_count = 0
for joint_num in range(heatmap_resized.shape[2]):
if h_count < 4:
tmp_concat_img = np.concatenate((tmp_concat_img, heatmap_resized[:, :, joint_num]), axis=1) \
if tmp_concat_img is not None else heatmap_resized[:, :, joint_num]
h_count += 1
else:
output_img = np.concatenate((output_img, tmp_concat_img), axis=0) if output_img is not None else tmp_concat_img
tmp_concat_img = None
h_count = 0
# last row img
if h_count != 0:
while h_count < 4:
tmp_concat_img = np.concatenate((tmp_concat_img, np.zeros(shape=(input_size, input_size), dtype=np.float32)), axis=1)
h_count += 1
output_img = np.concatenate((output_img, tmp_concat_img), axis=0)
# adjust heatmap color
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
return output_img
def getComposedFrame(self, frameNum):
""" Get a composition of all the modalities for a given frame """
# get sample modalities
rgb=self.getRGB(frameNum)
depthValues=self.getDepth(frameNum)
user=self.getUser(frameNum)
skel=self.getSkeletonImage(frameNum)
# Build depth image
depth = depthValues.astype(numpy.float32)
depth = depth*255.0/float(self.data['maxDepth'])
depth = depth.round()
depth = depth.astype(numpy.uint8)
depth = cv2.applyColorMap(depth,cv2.COLORMAP_JET)
# Build final image
compSize1=(max(rgb.shape[0],depth.shape[0]),rgb.shape[1]+depth.shape[1])
compSize2=(max(user.shape[0],skel.shape[0]),user.shape[1]+skel.shape[1])
comp = numpy.zeros((compSize1[0]+ compSize2[0],max(compSize1[1],compSize2[1]),3), numpy.uint8)
# Create composition
comp[:rgb.shape[0],:rgb.shape[1],:]=rgb
comp[:depth.shape[0],rgb.shape[1]:rgb.shape[1]+depth.shape[1],:]=depth
comp[compSize1[0]:compSize1[0]+user.shape[0],:user.shape[1],:]=user
comp[compSize1[0]:compSize1[0]+skel.shape[0],user.shape[1]:user.shape[1]+skel.shape[1],:]=skel
return comp
def Visualize(self,img_path, output_path):
original_img = cv2.imread(img_path, 1)
width, height, _ = original_img.shape
#Reshape to the network input shape (3, w, h).
#img = np.array([np.transpose(np.float32(original_img), (2, 0, 1))])
#Get the 512 input weights to the softmax.
class_weights = self.model.layers[-1].get_weights()[0]
final_conv_layer = self.get_output_layer(self.model, "conv2d_26")
get_output = K.function([self.model.layers[0].input], \
[final_conv_layer.output,
self.model.layers[-1].output])
[conv_outputs, predictions] = get_output([np.array([original_img])])
conv_outputs = conv_outputs[0, :, :, :]
print(predictions)
#Create the class activation map.
cam = np.ones(conv_outputs.shape[0 : 2], dtype = np.float32)
target_class = 1
for i, w in enumerate(class_weights[:, target_class]):
cam+= w * conv_outputs[:, :,i]
print("predictions", predictions)
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
print(cam.shape)
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.CV_8UC1)
heatmap[np.where(cam < 0.2)] = 0
img = heatmap*0.5 + original_img
cv2.imwrite(output_path, img)
def recalculate(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
section = slice(in_mark,out_mark)
# calc heatmaps
for s in self.surfaces:
if s.defined:
s.generate_heatmap(section)
# calc distirbution accross all surfaces.
results = []
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
results.append(len(gaze_on_srf))
self.metrics_gazecount = len(gaze_on_srf)
if results == []:
logger.warning("No surfaces defined.")
return
max_res = max(results)
results = np.array(results,dtype=np.float32)
if not max_res:
logger.warning("No gaze on any surface for this section!")
else:
results *= 255./max_res
results = np.uint8(results)
results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
for s,c_map in zip(self.surfaces,results_c_maps):
heatmap = np.ones((1,1,4),dtype=np.uint8)*125
heatmap[:,:,:3] = c_map
s.metrics_texture = Named_Texture()
s.metrics_texture.update_from_ndarray(heatmap)
def getComposedFrameOverlapUser(self, frameNum):
""" Get a composition of all the modalities for a given frame """
# get sample modalities
rgb=self.getRGB(frameNum)
depthValues=self.getDepth(frameNum)
user=self.getUser(frameNum)
mask = numpy.mean(user, axis=2) > 150
mask = numpy.tile(mask, (3,1,1))
mask = mask.transpose((1,2,0))
# Build depth image
depth = depthValues.astype(numpy.float32)
depth = depth*255.0/float(self.data['maxDepth'])
depth = depth.round()
depth = depth.astype(numpy.uint8)
depth = cv2.applyColorMap(depth,cv2.COLORMAP_JET)
# Build final image
compSize=(max(rgb.shape[0],depth.shape[0]),rgb.shape[1]+depth.shape[1])
comp = numpy.zeros((compSize[0]+ compSize[0],max(compSize[1],compSize[1]),3), numpy.uint8)
# Create composition
comp[:rgb.shape[0],:rgb.shape[1],:]=rgb
comp[:depth.shape[0],rgb.shape[1]:rgb.shape[1]+depth.shape[1],:]= depth
comp[compSize[0]:compSize[0]+user.shape[0],:user.shape[1],:]= mask * rgb
comp[compSize[0]:compSize[0]+user.shape[0],user.shape[1]:user.shape[1]+user.shape[1],:]= mask * depth
return comp
def create_scale_bar(self, width, height):
# Create scale bar
gradient_pane = np.zeros((height, width, 3), dtype=np.uint8)
gradient_pane[:] = 255
bar_height = 10
gradient = np.repeat(np.linspace(0, 255, width-100)[np.newaxis,:], bar_height, axis=0)
imout = cv2.normalize(gradient, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8UC1)
gradient_clr = cv2.applyColorMap(imout, cv2.COLORMAP_JET)
xd = (width - gradient.shape[1]) / 2
yd = 10
gradient_pane[yd:gradient.shape[0]+yd, xd:gradient.shape[1]+xd, :] = gradient_clr
font = cv2.FONT_HERSHEY_SIMPLEX
#font = cv2.FONT_HERSHEY_PLAIN
step = gradient.shape[1] / 10.0
for i in range(0, 11):
x = int(xd+i*step)
cv2.line(gradient_pane, (x, yd+5), (x, yd+10), (0,0,0), 1)
cv2.putText(gradient_pane, str(i*10), (x-5, yd+30), font, 0.5, (0, 0, 0), 1)
xd = int((width - 200) / 2)
cv2.putText(gradient_pane, "Confidence in %", (xd, 80), font, 0.6, (0, 0, 0), 1)
return gradient_pane
def grad_cam(input_model, model_x, orig_x, category_index, layer_name, class_names):
output = input_model.output
final_layer = Lambda(lambda x: target_category_loss(x, category_index, len(class_names)))
output = final_layer(output)
model = Model(inputs=input_model.input, outputs=output)
loss = K.sum(model.layers[-1].output)
conv_output = model.get_layer(layer_name).output
grads = normalize(K.gradients(loss, conv_output)[0])
gradient_function = K.function([model.layers[0].input, K.learning_phase()], [conv_output, grads])
output, grads_val = gradient_function([model_x, 0])
output, grads_val = output[0, :], grads_val[0, :, :, :]
weights = np.mean(grads_val, axis=(0, 1))
cam = np.zeros(output.shape[0: 2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * output[:, :, i]
cam = np.maximum(cam, np.zeros(output.shape[0: 2], dtype=np.float32))
cam = cam.squeeze()
cam = cv2.applyColorMap(np.uint8(255 * cam / np.max(cam)), cv2.COLORMAP_JET)
cam = cv2.resize(cam, (np.shape(orig_x)[0], np.shape(orig_x)[1]))
cam = 0.4 * cam + 0.6 * orig_x
return np.uint8(cam)
def main():
normalize = trans.Normalize(mean=[0.4001, 0.4401, 0.4687],
std=[0.229, 0.224, 0.225])
transform = trans.Compose([
trans.Scale((224,224)),
trans.ToTensor(),
normalize,
])
classes = {int(key): value for (key, value)
in parse_json(configs.class_info_dir).items()}
vgg_cam = models.vgg_cam()
vgg_cam = vgg_cam.cuda()
checkpoint = torch.load(configs.best_ckpt_dir)
vgg_cam.load_state_dict(checkpoint['state_dict'])
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
finalconv_name = 'classifier' # this is the last conv layer of the network
vgg_cam._modules.get(finalconv_name).register_forward_hook(hook_feature)
# get the softmax weight
params = list(vgg_cam.parameters())
weight_softmax = np.squeeze(params[-1].data.cpu().numpy())
img_path = 'playing_guitar_023.jpg'
save_fig_dir = 'cam_' + img_path
img_pil = Image.open(img_path)
img_tensor = transform(img_pil)
img_variable = Variable(img_tensor.unsqueeze(0).cuda())
transformed_img = img_variable.data.cpu().numpy()[0]
transformed_img = untransform(transformed_img)
outputs, _ = vgg_cam(img_variable)
h_x = F.softmax(outputs).data.squeeze()
probs, idx = h_x.sort(0, True)
top_number = 5
prob = probs.cpu().numpy()[:top_number]
idx_ = idx.cpu().numpy()[:top_number]
OUT_CAM = returnCAM(features_blobs[-1],weight_softmax,idx_,prob)
plt.figure(1, figsize=(8, 6))
ax = plt.subplot(231)
ax.imshow(transformed_img[:,:,(2,1,0)])
for b_index, (idx,prob_in,cam) in enumerate(zip(idx_,prob,OUT_CAM)):
cl = str(classes[idx])
height, width, _ = transformed_img.shape
heatmap = cv2.applyColorMap(cv2.resize(cam, (width, height)), cv2.COLORMAP_JET)
result = heatmap * 0.3 + transformed_img * 0.7
ax = plt.subplot(2,3,b_index+2)
ax.imshow(result.astype(np.uint8)[:,:,(2,1,0)])
ax.set_title(('{}:{}').format(cl,('%.3f' % prob_in)), fontsize=8)
plt.savefig(save_fig_dir)
def recalculate(self):
in_mark = self.g_pool.trim_marks.in_mark
out_mark = self.g_pool.trim_marks.out_mark
section = slice(in_mark,out_mark)
# calc heatmaps
for s in self.surfaces:
if s.defined:
s.generate_heatmap(section)
# calc metrics:
gaze_in_section = list(chain(*self.g_pool.positions_by_frame[section]))
results = []
for s in self.surfaces:
gaze_on_srf = s.gaze_on_srf_in_section(section)
results.append(len(gaze_on_srf))
self.metrics_gazecount = len(gaze_on_srf)
max_res = max(results)
results = np.array(results,dtype=np.float32)
if not max_res:
logger.warning("No gaze on any surface for this section!")
else:
results *= 255./max_res
results = np.uint8(results)
results_c_maps = cv2.applyColorMap(results, cv2.COLORMAP_JET)
for s,c_map in zip(self.surfaces,results_c_maps):
heatmap = np.ones((1,1,4),dtype=np.uint8)*125
heatmap[:,:,:3] = c_map
s.metrics_texture = create_named_texture(heatmap)
def energycalc(self, param, eye2, eyecontour, showimage=False):
x, y, r = param
mask = self.circularmask(x, y, r)
eyein = mask*eye2
energyin = np.sum(eyein)
mask *= 255
mask = np.array(mask,dtype=np.uint8)
maskcontour = cv2.Canny(mask,100,200)
energyborder = -(np.sum(eyecontour*maskcontour)/np.sum(maskcontour))
if showimage:
scale = 4
# eye2
img1 = eye2/(2*max(eye2.max(),-eye2.min())) + .5 # bring eye2 between 0 and 1
img1 = img1*.6 + np.greater(img1,.5)*.4 # move values away from .5
img1 = (img1*255).astype('uint8')
# img1 = imresize(img1, (self.ny*scale, self.nx*scale))
img1 = np.repeat(np.repeat(img1, scale, axis=0), scale, axis=1)
cv2.circle(img1, (int(x*scale), int(y*scale)), int(r*scale), 128, 1)
# eyecontour
img2 = (eyecontour*(255./self.maxcontour)).astype('uint8')
# img2 = imresize(img2, (self.ny*scale, self.nx*scale))
img2 = np.repeat(np.repeat(img2, scale, axis=0), scale, axis=1)
cv2.circle(img2, (int(x*scale), int(y*scale)), int(r*scale), 0, 1)
# display both
img = np.hstack((img1, img2))
img = cv2.applyColorMap(img, cv2.COLORMAP_JET)
cv2.imshow('preproc', img)
energy = (1-self.alpha)*energyin + self.alpha*energyborder
return energy
def read_and_generate_heatmap(input_path, output_path):
original_img = cv2.imread(input_path).astype(np.float32)
width, height, _ = original_img.shape
im = process_image(cv2.resize(original_img,(224,224)))
out = model.predict(im)
class_weights = model.layers[-1].get_weights()[0]
top_3_hits = np.argsort(out[0], axis=1)[0][::-1][:3]
print("predictions", top_3_hits)
[print(semantics.ix[hit + 1].semantic) for hit in top_3_hits]
conv_output = out[1][0,:,:,:]
#Create the class activation map.
cam = np.zeros(dtype = np.float32, shape = conv_output.shape[1:3])
class_to_visualize = np.argmax(out[0]) # visualise best class
for i, w in enumerate(class_weights[:, class_to_visualize]):
cam += w * conv_output[i, :, :]
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
heatmap[np.where(cam < 0.2)] = 0
temp = heatmap*0.5 + original_img
cv2.imwrite(output_path, temp)
def saveBlendedImages(image_data1,image_data2, folder_output):
"""
Saves blended image stacks to PNG files
:param image_data1: images stack (3D nparray)
:param image_data2: images stack (3D nparray)
:param folder_output: output folder
"""
if not os.path.exists(os.path.join(folder_output,'PET_MRI_results')):
os.makedirs(os.path.join(folder_output,'PET_MRI_results'))
rows, cols, num_imgs = image_data1.shape
for i in range(num_imgs):
rows1,cols1 = image_data2[:,:,i].shape
rows2,cols2 = image_data1[:,:,i].shape
size = (max(rows1,rows2),max(cols1,cols2))
img1_filled = imgp.fillImg(image_data2[:,:,i],size)
img2_filled = imgp.fillImg(image_data1[:,:,i],size)
img_show1 = ((img1_filled.astype(np.float) - np.min(img1_filled))/np.max(img1_filled))*255
img_show2 = ((img2_filled.astype(np.float) - np.min(img2_filled))/np.max(img2_filled))*255
img1_bgr = cv2.cvtColor(np.uint8(img_show1), cv2.COLOR_GRAY2BGR)
img2_bgr = cv2.cvtColor(np.uint8(img_show2), cv2.COLOR_GRAY2BGR)
img2_bgr = cv2.applyColorMap(img2_bgr, cv2.COLORMAP_HOT)
dst = cv2.addWeighted(img1_bgr, 0.5, img2_bgr, 0.5, 0)
cv2.imwrite(os.path.join(folder_output,'PET_MRI_results', 'Img_fusion_' + str(i) + '.png'), dst)
def ShowFrame ( self, frames ): show_frames = [] for i in range(N_CAMS): show_frames.append( cv2.resize(frames[i], (HEIGHT,WIDTH)) ) #np.copy(frames[i]) ) # Graficar perfiles y centroides self.plots[i].CalculateProfiles( show_frames[i] ) # Pasar a 3 canales (para poder mostrar el cursor de color) if type(self.colorMap) is long: show_frames[i] = cv2.applyColorMap(show_frames[i], self.colorMap) # Aplicar color map show_frames[i] = cv2.cvtColor(show_frames[i], cv2.COLOR_BGR2RGB) # Pasar de BGR a RGB else: if self.colorMap == "invert": # Invertir escala show_frames[i] = 255 - show_frames[i] show_frames[i] = cv2.cvtColor(show_frames[i], cv2.COLOR_GRAY2RGB) # Pasar de gris a RGB # Draw regions of interest and centroid CM = np.array(self.plots[i].CM, dtype=np.int16) size = self.ROI_sizes[i] pos = self.ROI_positions[i] pt1 = ( pos[0]-size/2, pos[1]-size/2 ) pt2 = ( pos[0]+size/2, pos[1]+size/2 ) cv2.rectangle(show_frames[i], pt1, pt2, (0,255,0), 2 ) pt1 = ( CM[0], 0 ) pt2 = ( CM[0], HEIGHT ) pt3 = ( 0, CM[0] ) pt4 = ( WIDTH, CM[0] ) cv2.line(show_frames[i], pt1, pt2, (0,0,255), 2 ) cv2.line(show_frames[i], pt3, pt4, (0,0,255), 2 ) self.bmps[i] = wx.BitmapFromBuffer(self.orig_width, self.orig_height, show_frames[i]) self.ImgControls[i].SetBitmap(self.bmps[i])
def create_mask_montage(self, image, predictions):
"""
Create a montage showing the probability heatmaps for each one one of the
detected objects
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `mask`.
"""
masks = predictions.get_field("mask")
masks_per_dim = self.masks_per_dim
masks = torch.nn.functional.interpolate(
masks.float(), scale_factor=1 / masks_per_dim
).byte()
height, width = masks.shape[-2:]
max_masks = masks_per_dim ** 2
masks = masks[:max_masks]
# handle case where we have less detections than max_masks
if len(masks) < max_masks:
masks_padded = torch.zeros(max_masks, 1, height, width, dtype=torch.uint8)
masks_padded[: len(masks)] = masks
masks = masks_padded
masks = masks.reshape(masks_per_dim, masks_per_dim, height, width)
result = torch.zeros(
(masks_per_dim * height, masks_per_dim * width), dtype=torch.uint8
)
for y in range(masks_per_dim):
start_y = y * height
end_y = (y + 1) * height
for x in range(masks_per_dim):
start_x = x * width
end_x = (x + 1) * width
result[start_y:end_y, start_x:end_x] = masks[y, x]
return cv2.applyColorMap(result.numpy(), cv2.COLORMAP_JET)
def write_image(image, name, scale=True, apply_color=False):
tmp = image.copy()
if scale:
tmp = scale_to_img(tmp)
if apply_color:
tmp = cv2.applyColorMap(tmp, cv2.COLORMAP_JET)
cv2.imwrite(os.path.join(output_dir, name), tmp)
def convert_depth_image_to_jetrgb(image_path):
dimage = cv2.imread(image_path, -1) # load in raw mode
min_value = dimage.min()
max_value = dimage.max()
im_range = (dimage.astype('float32') - min_value) / (max_value - min_value)
im_range = 255.0 * im_range
out_img = cv2.applyColorMap(im_range.astype("uint8"), cv2.COLORMAP_JET)
return Image.fromarray(out_img)
def get_img_heatmap(orig_img, activation_map):
"""Draw a heatmap on top of the original image using intensities from activation_map"""
heatmap = cv2.applyColorMap(activation_map, cv2.COLORMAP_COOL)
heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
img_heatmap = np.float32(heatmap) + np.float32(orig_img)
img_heatmap = img_heatmap / np.max(img_heatmap)
img_heatmap *= 255
return img_heatmap.astype(int)
def SGD(self, training_data, epochs, mini_batch_size, eta, test_data=None):
"""Train the neural network using mini-batch stochastic gradient descent. The "training_data" is a list of tuples "(x, y)" representing the training inputs and the desired outputs. The other non-optional parameters are self-explanatory. If "test_data" is provided then the network will be evaluated against the test data after each epoch, and partial progress printed out. This is useful for tracking prorgress, but slows things down substantially."""
if test_data: n_test=len(test_data)
n = len(training_data)
for j in xrange(epochs):
random.shuffle(training_data)
mini_batches= [
training_data[k:k+mini_batch_size]
for k in xrange(0, n, mini_batch_size)]
for mini_batch in mini_batches:
self.update_mini_batch(mini_batch, eta)
#display weights and biases after every training set
wei = self.weights[0]
wei = self.scaleGSImg(wei)
# now wei is uint8 matrix
l =[] # l is weights matrix
for i in xrange(self.sizes[1]):
l.append(wei[i].reshape(28,28))
'''l[i] = cv2.resize(l[i],None,fx=10,fy=10, interpolation = cv2.INTER_LINEAR)
cv2.imshow('Output'+str(i),l[i])'''
# transform those images using weights
linscal = self.weights[1] # get corresponding weights for second layer
for i in xrange(10):
img = 0
for k in xrange(self.sizes[1]):
img = img+linscal[i][k] * l[k]
img = self.scaleGSImg(img)
img = cv2.resize(img,None,fx=4,fy=4,interpolation = cv2.INTER_LINEAR)
img = cv2.applyColorMap(img,cv2.COLORMAP_HSV)
cv2.imshow('layer'+str(i),img)
cv2.waitKey(10)
#wei = cv2.resize(wei,None,fx=1,fy=1, interpolation = cv2.INTER_LINEAR)
wei = cv2.applyColorMap(wei,cv2.COLORMAP_HSV)
cv2.imshow('weights',wei)
#time.sleep(1)
if test_data:
print "Epoch {0}: {1} / {2}".format(j, self.evaluate(test_data), n_test)
else:
print "Epoch {0} complete".format(j)
def downloadVideoAndEntropy(f, frame):
fhandle = open(f, 'ab')
ftp.retrbinary('RETR ' + f, fhandle.write)
cap = cv2.VideoCapture(f)
if cap.isOpened():
cap.set(1, frame)
ret, newframe = cap.read()
if ret:
newname = f.split(".")[0]
finalname = newname + '_frame_' + str(frame) + '.png'
cv2.imwrite(finalname, newframe)
colorIm = Image.open(finalname)
greyIm = colorIm.convert('L')
colorIm = np.array(colorIm)
greyIm = np.array(greyIm)
N = 5
S = greyIm.shape
E = np.array(greyIm)
for row in range(S[0]):
for col in range(S[1]):
Lx = np.max([0, col - N])
Ux = np.min([S[1], col + N])
Ly = np.max([0, row - N])
Uy = np.min([S[0], row + N])
region = greyIm[Ly:Uy, Lx:Ux].flatten()
E[row, col] = entropy(region)
grayImage = cv2.applyColorMap(greyIm, cv2.COLOR_BGR2GRAY)
entropyImage = cv2.applyColorMap(greyIm, cv2.COLORMAP_JET)
cv2.imwrite('gray_' + finalname, greyIm)
# cv2.imwrite('color_' + finalname, entropyImage)
a = np.empty_like(E)
a[:, :] = E
a = np.flipud(a)
fig = plt.figure()
plt.imshow(a, cmap=plt.get_cmap("jet"), origin="lower")
plt.xlabel('Entropy in 10x10 neighborhood')
plt.colorbar()
plt.savefig('color_' + finalname, bbox_inches='tight')
plt.imshow(E, cmap=plt.get_cmap("jet"), origin="lower")
plt.plot()
htmlname = newname + '_frame_' + str(frame) + '.html'
mpld3.save_html(fig, htmlname)
# mpld3.fig_to_html(fig, template_type="simple")
print 'finished!'
cap.release()
def show_gt(img,gt,boxes=None):
gth, gtw = gt.shape[:2]
mask = gt
height, width = img.shape[:2]
gtR = cv2.resize(mask.astype(np.float32),(width, height))
fake = cv2.applyColorMap(gtR.astype(np.uint8)*30, cv2.COLORMAP_HSV);
fake[np.where(gtR==0)]=0
res = img*0.8 + fake*0.2
return res
def changeColorSet(imageName):
"""Takes image name as input and changes the color"""
image = cv2.imread(imageName)
cv2.imshow("INPUT", image)
for i in xrange(0,12):
res = cv2.applyColorMap(image, i)
cv2.imshow("OUTPUT", res)
cv2.waitKey(0)
def generate_heatmap(self,section):
if self.cache is None:
logger.warning('Surface cache is not build yet.')
return
x,y = self.real_world_size['x'],self.real_world_size['y']
x = max(1,int(x))
y = max(1,int(y))
filter_size = int(int(self.heatmap_detail * x)/2)*2 +1
std_dev = int(filter_size /6.)
self.heatmap = np.ones((y,x,4),dtype=np.uint8)
all_gaze = []
for frame_idx,c_e in enumerate(self.cache[section]):
if c_e:
frame_idx+=section.start
for gp in self.gaze_on_srf_by_frame_idx(frame_idx,c_e['m_from_screen']):
if gp['confidence']>=self.g_pool.min_data_confidence:
all_gaze.append(gp['norm_pos'])
if not all_gaze:
logger.warning("No gaze data on surface for heatmap found.")
all_gaze.append((-1.,-1.))
all_gaze = np.array(all_gaze)
all_gaze *= [self.real_world_size['x'],self.real_world_size['y']]
hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1],
bins=[x,y],
range=[[0, self.real_world_size['x']], [0,self.real_world_size['y']]],
normed=False,
weights=None)
hist = np.rot90(hist)
#smoothing..
hist = cv2.GaussianBlur(hist,(filter_size,filter_size),std_dev)
maxval = np.amax(hist)
if maxval:
scale = 255./maxval
else:
scale = 0
hist = np.uint8( hist*(scale) )
#colormapping
c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET)
self.heatmap[:,:,:3] = c_map
self.heatmap[:,:,3] = 125
self.heatmap_texture = Named_Texture()
self.heatmap_texture.update_from_ndarray(self.heatmap)
def colorize_depth(img):
# scale the value from 0 to 255
img = img.astype(float)
img *= 255 / img.max()
img = img.astype(np.uint8)
# colorize depth map
res = cv2.applyColorMap(img, cv2.COLORMAP_JET)
#res = cv2.cvtColor(res, cv2.COLOR_BGR2RGB)
#plt.imsave('tmp.png', res)
return res
def saveCompareMatrix(matrix, filename):
"""
Function saves matrix with similarity measure for two stack of images stored in matrix
:param matrix: similarity matrix
:param filename: output filename
"""
img_save = (matrix.astype(np.float) - np.min(matrix))/np.max(matrix)*255
img_bgr = cv2.cvtColor(np.uint8(img_save), cv2.COLOR_GRAY2BGR)
colormap_img = cv2.applyColorMap(img_bgr, cv2.COLORMAP_JET)
cv2.imwrite(filename, colormap_img)
def createColorMap(srcImg, min=0.0, max=0.0):
dstImg8U = (srcImg-min) * (255./(max-min))
dstImg8U = dstImg8U.astype(np.uint8)
colorMapImg = cv2.applyColorMap(dstImg8U, cv2.COLORMAP_JET)
srcImg32f = srcImg.astype(np.float32)
retVal, maskImg = cv2.threshold(srcImg32f, min, 255, cv2.THRESH_BINARY)
maskImg = maskImg.astype(np.uint8)
return colorMapImg, maskImg
def visualize_votes(self, map_pred, map_gt, mask_gt, epoch, i_batch):
img_dir = os.path.join(self.args.save_dir, 'image', str(self.args.lr))
if not os.path.exists(img_dir):
os.makedirs(img_dir)
map_pred = map_pred.detach().cpu().numpy()
map_gt = map_gt.detach().cpu().numpy()
mask = np.uint8(mask_gt.detach().cpu().numpy()[0])
image = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint8)
ys, xs = np.nonzero(mask)
# visualize pred
images_pred = [image.copy() for _ in range(map_pred.shape[0] // 2)]
for i_pt in range(len(ys)):
for i_keypt in range(map_pred.shape[0] // 2):
y = ys[i_pt]
x = xs[i_pt]
map_x = map_pred[i_keypt * 2, y, x]
map_y = map_pred[i_keypt * 2 + 1, y, x]
if map_x == 0:
continue
angle = np.arctan(
np.abs(map_y) / np.abs(map_x)) / (np.pi / 2) * 90
if map_x < 0 and map_y > 0:
angle = 180 - angle
if map_x < 0 and map_y < 0:
angle = 180 + angle
if map_x >= 0 and map_y < 0:
angle = 360 - angle
images_pred[i_keypt][y, x] = int(round(angle / 360 * 255))
images_pred = [
cv2.applyColorMap(im_gray, cv2.COLORMAP_HSV)
for im_gray in images_pred
]
for i, im in enumerate(images_pred):
im[mask == 0] = (0, 0, 0)
img_pred_name = os.path.join(
img_dir, '{}_{}_vote_kp_{}_pred.jpg'.format(epoch, i_batch, i))
cv2.imwrite(img_pred_name, im)
# visualize gt
images_gt = [image.copy() for _ in range(map_gt.shape[0] // 2)]
for i_pt in range(len(ys)):
for i_keypt in range(map_gt.shape[0] // 2):
y = ys[i_pt]
x = xs[i_pt]
map_x = map_gt[i_keypt * 2, y, x]
map_y = map_gt[i_keypt * 2 + 1, y, x]
if map_x == 0:
continue
angle = np.arctan(
np.abs(map_y) / np.abs(map_x)) / (np.pi / 2) * 90
if map_x < 0 and map_y > 0:
angle = 180 - angle
if map_x < 0 and map_y < 0:
angle = 180 + angle
if map_x >= 0 and map_y < 0:
angle = 360 - angle
images_gt[i_keypt][y, x] = int(round(angle / 360 * 255))
images_gt = [
cv2.applyColorMap(im_gray, cv2.COLORMAP_HSV)
for im_gray in images_gt
]
for i, im in enumerate(images_gt):
im[mask == 0] = (0, 0, 0)
img_gt_name = os.path.join(
img_dir, '{}_{}_vote_kp_{}_gt.jpg'.format(epoch, i_batch, i))
cv2.imwrite(img_gt_name, im)
def main():
# ==============================================================================
#1. Setup, file paths
# ==============================================================================
#Final poultry model
poultry_model_version = 'final_poultry_model'
#Final swine model
swine_model_version = 'final_swine_model'
#Directory for data
data_dir = '../data'
#Image directory
image_dir = os.path.join(data_dir, 'images', 'fig3')
#Figure directory
fig_dir = '../figures'
#Model directory
model_dir = '../models'
# Paths to saved models
poultry_graph_path = os.path.join(model_dir, poultry_model_version + '.pb')
swine_graph_path = os.path.join(model_dir, swine_model_version + '.pb')
#Location of CAM results
cam_results_csv = os.path.join(data_dir, 'csv', 'cam_results.csv')
score_csv = os.path.join(data_dir, 'csv', 'rescored_images.csv')
#PIL fonts
bold_font = ImageFont.truetype(
os.path.join(data_dir, 'fonts/DejaVuSans-Bold.ttf'), 16)
font = ImageFont.truetype(os.path.join(data_dir, 'fonts/DejaVuSans.ttf'),
16)
#Image of interest
image_id = 'north-carolina_duplin_11801_287_6_1_0_18_-944_13554'
# ==============================================================================
#2. Load model graphs
# ==============================================================================
print("Loading model graphs...")
#Reset graph params
tf.reset_default_graph()
poultry_graph = tf.Graph()
swine_graph = tf.Graph()
# Load in poultry model
with poultry_graph.as_default():
with tf.gfile.FastGFile(poultry_graph_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def)
#Load in swine model
with swine_graph.as_default():
with tf.gfile.FastGFile(swine_graph_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def)
# ==============================================================================
#3. Prepare CAM results for plotting
# ==============================================================================
print("Loading CAM results...")
#CAM results
square_footage_df = pd.read_csv(cam_results_csv,
header=None,
names=[
'image_id', 'cluster', 'center',
'easting', 'northing', 'zone',
'square_footage', 'activation_bounds',
'pixel_cluster_x', 'pixel_cluster_y'
])
#Add image cluster id
square_footage_df['image_cluster_id'] = [
str(square_footage_df['image_id'][i]) + '_' +
str(square_footage_df['cluster'][i]) + '.jpeg'
for i in range(len(square_footage_df))
]
#Re-scored images
new_score_df = pd.read_csv(
score_csv,
header=None,
names=['image_cluster_id', 'image_id', 'poultry_score', 'swine_score'])
#Merge together
cam_merge = square_footage_df.merge(new_score_df, on='image_cluster_id')
# ==============================================================================
#4. Score original image
# ==============================================================================
print("Scoring original image...")
#Get path to original image
img_path = os.path.join(image_dir, image_id + '.jpeg')
#Read in image
image_data = gfile.FastGFile(img_path, 'rb').read()
with tf.Session(graph=poultry_graph) as sess:
#Get final weights
softmax_tensor = sess.graph.get_tensor_by_name('import/final_result:0')
#Get model prediction for image
poultry_predictions = sess.run(softmax_tensor, \
{'import/DecodeJpeg/contents:0': image_data,
'import/final_training_ops/dropout/Placeholder:0':1.0})
poultry_score = poultry_predictions[0][1]
with tf.Session(graph=swine_graph) as sess:
#Get final weights
softmax_tensor = sess.graph.get_tensor_by_name('import/final_result:0')
#Get model prediction for image
swine_predictions = sess.run(softmax_tensor, \
{'import/DecodeJpeg/contents:0': image_data,
'import/final_training_ops/dropout/Placeholder:0':1.0})
swine_score = swine_predictions[0][1]
#Identify new scores on recentered image
new_poultry_score = cam_merge.loc[(cam_merge.image_id_x == image_id),
'poultry_score'].values[0]
new_swine_score = cam_merge.loc[(cam_merge.image_id_x == image_id),
'swine_score'].values[0]
# ==============================================================================
#5. Run poultry and swine CAMs on original image
# ==============================================================================
print("Re-running poultry and swine CAMs on image...")
#Run Poultry CAM on original image
with tf.Session(graph=poultry_graph) as sess:
#Get cam results
activated_pixels, pixel_clusters, pixel_labels, final_result = run_cam_loop(
img_path, sess)
##Combine with original image
img = cv2.imread(img_path)
heatmap = cv2.applyColorMap(final_result, cv2.COLORMAP_JET)
result_img = heatmap * 0.3 + img * 0.5
#Path to results
outpath = os.path.join(image_dir, image_id + '_poultry.jpeg')
#Write and read in
cv2.imwrite(outpath, result_img)
#Path to recentered image
img_path = os.path.join(image_dir, image_id + '_0.jpeg')
#Run Swine CAM on recentered image
with tf.Session(graph=swine_graph) as sess:
#Get cam results
activated_pixels, pixel_clusters, pixel_labels, final_result = run_cam_loop(
img_path, sess)
##Combine with original image
img = cv2.imread(img_path)
heatmap = cv2.applyColorMap(final_result, cv2.COLORMAP_JET)
result_img = heatmap * 0.3 + img * 0.5
#Path to results
outpath = os.path.join(image_dir, image_id + '_swine.jpeg')
#Write and read in
cv2.imwrite(outpath, result_img)
# ==============================================================================
#5. Create full image
# ==============================================================================
print("Compiling figure...")
#Initialize image
new_im = PILImage.new('RGB', (299 * 2, 299 * 2), color=(255, 255, 255))
#Original image
org_im = PILImage.open(os.path.join(image_dir,
image_id + '.jpeg')).convert('RGB')
draw_image = ImageDraw.Draw(org_im)
draw_image.text((0, 0), 'a', 'white', font=bold_font)
new_im.paste(org_im, (0, 0))
#Poultry activation, original image
org_act = PILImage.open(os.path.join(image_dir, image_id +
'_poultry.jpeg')).convert('RGB')
draw_image = ImageDraw.Draw(org_act)
draw_image.text((0, 0), 'b', 'white', font=bold_font)
draw_image.text((120, 10),
'Poultry score: ' + '{0:1.2f}'.format(poultry_score),
'white',
font=bold_font)
draw_image.text((130, 30),
'Swine score: ' + '{0:1.2f}'.format(swine_score),
'white',
font=bold_font)
new_im.paste(org_act, (299, 0))
#Centered image
cent_im = PILImage.open(os.path.join(image_dir,
image_id + '_0.jpeg')).convert('RGB')
draw_image = ImageDraw.Draw(cent_im)
draw_image.text((0, 0), 'c', 'white', font=bold_font)
new_im.paste(cent_im, (0, 299))
#Swine activation centered image
cent_act = PILImage.open(os.path.join(image_dir, image_id +
'_swine.jpeg')).convert('RGB')
draw_image = ImageDraw.Draw(cent_act)
draw_image.text((0, 0), 'd', 'white', font=bold_font)
draw_image.text((120, 10),
'Poultry score: ' + '{0:1.2f}'.format(new_poultry_score),
'white',
font=bold_font)
draw_image.text((130, 30),
'Swine score: ' + '{0:1.2f}'.format(new_swine_score),
'white',
font=bold_font)
new_im.paste(cent_act, (299, 299))
#Save image
new_im.save(os.path.join(fig_dir,
'3_Figure_fgcam_CAM_Algorithm_Illustration.png'),
dpi=(300, 300))
print("Done.")
if inRight is not None:
frame = cv2.flip(inRight.getCvFrame(), 1)
if flipRectified:
frame = cv2.flip(frame, 1)
if inManip is not None:
frameManip = inManip.getCvFrame()
if inDisparity is not None:
# Flip disparity frame, normalize it and apply color map for better visualization
frameDisparity = inDisparity.getFrame()
if flipRectified:
frameDisparity = cv2.flip(frameDisparity, 1)
frameDisparity = (frameDisparity*disparity_multiplier).astype(np.uint8)
frameDisparity = cv2.applyColorMap(frameDisparity, cv2.COLORMAP_JET)
if inDet is not None:
detections = inDet.detections
if frame is not None:
for detection in detections:
bbox = frameNorm(croppedFrame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
bbox[::2] += offsetX
cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (255, 0, 0), 2)
cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
cv2.imshow("right", frame)
if frameDisparity is not None:
for detection in detections:
import os
import cv2
import numpy as np
files = os.listdir('../Plots/')
for file in files:
im = cv2.imread('../Plots/' + file, 0)
imC = cv2.applyColorMap(im, cv2.COLORMAP_SUMMER)
cv2.imwrite('../Coloured/' + file, imC)
def main():
global depth_points
# Create Kinect object and initialize
kin = kinz.Kinect(resolution=720, wfov=True, binned=True)
depth_window_name = 'Click on the Depth image'
color_window_name = 'Mapped coordinates in Color'
cv2.namedWindow(color_window_name, cv2.WINDOW_AUTOSIZE)
cv2.namedWindow(depth_window_name, cv2.WINDOW_AUTOSIZE)
cv2.setMouseCallback(depth_window_name, mouse_event)
points_3d = []
prev_points_3d = []
while True:
if kin.get_frames(get_color=True,
get_depth=True,
get_ir=False,
align_depth=True):
color_data = kin.get_color_data()
depth_data = kin.get_depth_data()
depth_image = np.array(depth_data.buffer, copy=True)
color_image = np.array(color_data.buffer, copy=True)
color_image = cv2.cvtColor(color_image, cv2.COLOR_BGRA2BGR)
# Project depth image points to color image points
# depth_points is a list of value pairs [[xd1,yd1], [xd2,yd2], ...]
# color_points have the same format as depth_points
# map_coords_depth_to_color return [-1, -1] if there is no depth information
# at the desire location, or the transformation failed.
color_points = kin.map_coords_depth_to_color(depth_points)
# Backproject depth image points to 3D depth camera space
# points_3d is a list of triplets of X, Y, Z points in
# the depth camera reference
# if depth_coords is [[x1,y1], [x2,y2], ...]
# points_3d is [[X1, Y1, Z1], [X2, Y2, Z2], ...]
# points_3d contains [0, 0, 0] if for desired depth coordinates
# there is not depth available.
# E.g. if at [x2, y2] the depth is 0, points_3d = [[X1, Y1, Z1], [0, 0, 0]]
# we select 0 because depth = 0 means no depth data.
points_3d = kin.map_coords_depth_to_3D(depth_points)
if points_3d != prev_points_3d:
print("3D point:", points_3d)
# Draw mapped points in color image
for p in color_points:
cv2.circle(color_image, (p[0], p[1]), 8, (0, 0, 255), -1)
# Apply colormap on depth image for visualization
depth_colormap = cv2.applyColorMap(
cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)
# Draw depth points
for p in depth_points:
cv2.circle(depth_colormap, (p[0], p[1]), 5, (0, 0, 255), -1)
# Resize color image for visualization purposes
color_small = cv2.resize(color_image, None, fx=0.5, fy=0.5)
cv2.imshow(depth_window_name, depth_colormap)
cv2.imshow(color_window_name, color_small)
prev_points_3d = points_3d
k = cv2.waitKey(1) & 0xFF
if k == ord('c'):
depth_points = []
if k == 27:
break
kin.close() # close Kinect
cv2.destroyAllWindows()
def save_cam_to_image(img, mask, path):
heatmap = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
cam = heatmap + np.float32(img)
cam = cam / np.max(cam)
cv2.imwrite(path, np.uint8(255 * cam))
def save_batch_heatmaps(batch_image, batch_heatmaps, file_name,
normalize=True):
'''
batch_image: [batch_size, channel, height, width]
batch_heatmaps: ['batch_size, num_joints, height, width]
file_name: saved file name
'''
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_size = batch_heatmaps.size(0)
num_joints = batch_heatmaps.size(1)
heatmap_height = batch_heatmaps.size(2)
heatmap_width = batch_heatmaps.size(3)
grid_image = np.zeros((batch_size * heatmap_height,
(num_joints + 1) * heatmap_width,
3),
dtype=np.uint8)
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
for i in range(batch_size):
image = batch_image[i].mul(255) \
.clamp(0, 255) \
.byte() \
.permute(1, 2, 0) \
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255) \
.clamp(0, 255) \
.byte() \
.cpu().numpy()
resized_image = cv2.resize(image,
(int(heatmap_width), int(heatmap_height)))
height_begin = heatmap_height * i
height_end = heatmap_height * (i + 1)
for j in range(num_joints):
cv2.circle(resized_image,
(int(preds[i][j][0]), int(preds[i][j][1])),
1, [0, 0, 255], 1)
heatmap = heatmaps[j, :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
masked_image = colored_heatmap * 0.7 + resized_image * 0.3
cv2.circle(masked_image,
(int(preds[i][j][0]), int(preds[i][j][1])),
1, [0, 0, 255], 1)
width_begin = heatmap_width * (j + 1)
width_end = heatmap_width * (j + 2)
grid_image[height_begin:height_end, width_begin:width_end, :] = \
masked_image
# grid_image[height_begin:height_end, width_begin:width_end, :] = \
# colored_heatmap*0.7 + resized_image*0.3
grid_image[height_begin:height_end, 0:heatmap_width, :] = resized_image
cv2.imwrite(file_name, grid_image)
def color_mapping(mat):
norm = None
norm = cv.normalize(src=mat, dst=norm, alpha=0, beta=255,
norm_type=cv.NORM_MINMAX, dtype=cv.CV_8UC3)
return cv.applyColorMap(norm, cv.COLORMAP_HSV)
if enable_imu and not profile_frames:
print(
"imu frame {} in {} seconds: \n\taccel = {}, \n\tgyro = {}"
.format(
str(frame_count), str(frame_time - last_time),
str((acceleration_x, acceleration_y, acceleration_z)),
str((gyroscope_x, gyroscope_y, gyroscope_z))))
# Show images
if show_opencv_window and not profile_frames:
cv2.namedWindow('RealSense', cv2.WINDOW_AUTOSIZE)
if enable_rgb or enable_depth:
# make sure depth and color images have same number of channels so we can show them together in the window
if 3 == channels:
depth_colormap = cv2.applyColorMap(
cv2.convertScaleAbs(depth_image, alpha=0.03),
cv2.COLORMAP_JET) if enable_depth else None
else:
depth_colormap = cv2.cvtColor(
cv2.applyColorMap(
cv2.convertScaleAbs(depth_image, alpha=0.03),
cv2.COLORMAP_JET),
cv2.COLOR_RGB2GRAY) if enable_depth else None
# Stack both images horizontally
images = None
if enable_rgb:
images = np.hstack(
(color_image,
depth_colormap)) if enable_depth else color_image
elif enable_depth:
def save_batch_resized_heatmaps(batch_image,
batch_heatmaps,
file_name,
normalize=True,
toTensor=ToTensor(),
save=True):
'''
batch_image: [batch_size, channel, height, width]
batch_heatmaps: ['batch_size, num_joints, height, width]
file_name: saved file name
'''
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_heatmaps = F.interpolate(batch_heatmaps, 256, mode='nearest')
batch_size = batch_heatmaps.size(0)
num_joints = batch_heatmaps.size(1)
heatmap_height = batch_heatmaps.size(2)
heatmap_width = batch_heatmaps.size(3)
resized_height = 256
resized_width = 256
grid_image = np.zeros(
(batch_size * resized_height, (num_joints + 1) * resized_width, 3),
dtype=np.uint8)
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
for i in range(batch_size):
image = batch_image[i].mul(255)\
.clamp(0, 255)\
.byte()\
.permute(1, 2, 0)\
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
resized_image = cv2.resize(image,
(int(heatmap_width), int(heatmap_height)))
height_begin = resized_height * i
height_end = resized_height * (i + 1)
for j in range(num_joints):
cv2.circle(resized_image,
(int(preds[i][j][0]), int(preds[i][j][1])), 1,
[0, 0, 255], 1)
heatmap = heatmaps[j, :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
masked_image = colored_heatmap * 0.7 + resized_image * 0.3
cv2.circle(masked_image,
(int(preds[i][j][0]), int(preds[i][j][1])), 1,
[0, 0, 255], 1)
width_begin = resized_width * (j + 1)
width_end = resized_width * (j + 2)
masked_image = cv2.resize(masked_image,
(resized_width, resized_height))
grid_image[height_begin:height_end, width_begin:width_end, :] = \
masked_image
# grid_image[height_begin:height_end, width_begin:width_end, :] = \
# colored_heatmap*0.7 + resized_image*0.3
resized_image = cv2.resize(resized_image,
(resized_width, resized_height))
grid_image[height_begin:height_end, 0:resized_width, :] = resized_image
# print('brefore:', grid_image.shape)
grid_image = copy.deepcopy(grid_image[:, :, ::-1])
# print('after:', grid_image)
# out_image = toTensor(grid_image)
out_image = grid_image
if save is True:
cv2.imwrite(file_name, resized_image)
return out_image, resized_image
# 用原圖的 BGR 處理
y, x, h = re_ori.shape
img = np.zeros(re_ori.shape, np.uint8)
for j in range(y):
for i in range(x):
b, g, r = re_ori[j, i]
M = max(b, g, r)
if M != b or b < 85:
img[j, i] = [255, 255, 255]
else:
img[j, i] = re_ori[j, i]
# cv2.imshow("img", img)
# 用 heat 處理
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
heat = cv2.applyColorMap(gray, cv2.COLORMAP_JET)
# cv2.imwrite("./count/DJI_0002_heat.jpg", heat)
# cv2.imshow("heat", heat)
for j in range(y):
for i in range(x):
b, g, r = heat[j, i]
M = max(b, g, r)
if M == r:
img[j, i] = [255, 255, 255]
elif M != g:
img[j, i] = [255, 255, 255]
else:
img[j, i] = re_ori[j, i]
# cv2.imshow("img2", img)
logit = model.forward(input_img)
h_x = F.softmax(logit, 1).data.squeeze()
probs, idx = h_x.sort(0, True)
print('RESULT ON ' + img_name)
io_image = np.mean(labels_IO[idx[:10].numpy()])
if io_image < 0.5:
print('--TYPE OF ENVIRONMENT: indoor')
else:
print('--TYPE OF ENVIRONMENT: outdoor')
print('--SCENE CATEGORIES:')
for i in range(0, 5):
print('{:.3f} -> {}'.format(probs[i], classes[idx[i]]))
responses_attribute = W_attribute.dot(features_blobs[1])
idx_a = np.argsort(responses_attribute)
print('--SCENE ATTRIBUTES:')
print(', '.join([labels_attribute[idx_a[i]] for i in range(-1, -10, -1)]))
print('Class activation map is saved as cam.jpg')
CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0]])
img = cv2.imread('test.jpg')
height, width, _ = img.shape
heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (width, height)),
cv2.COLORMAP_JET)
result = heatmap * 0.4 + img * 0.5
cv2.imwrite('cam.jpg', result)
pooled_gradients = torch.mean(gradients, dim=[0, 2, 3])
# # get the activations of the last convolutional layer
activations = sn.get_activations(img).detach()
# weight the channels by corresponding gradients
for i in range(activations.size(1)):
activations[:, i, :, :] *= pooled_gradients[i]
# average the channels of the activations
heatmap = torch.mean(activations, dim=1).squeeze()
# relu on top of the heatmap
# expression (2) in https://arxiv.org/pdf/1610.02391.pdf
heatmap = np.maximum(heatmap, 0)
# normalize the heatmap
heatmap /= torch.max(heatmap)
# draw the heatmap
plt.matshow(heatmap.squeeze())
heatmap = heatmap.numpy()
plt.show()
img = cv2.imread('./data/Elephant/1.jpg')
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = heatmap * 0.4 + img
cv2.imwrite('./map.jpg', superimposed_img)
target = targets[i] # (shape: (h, w))
target[target > max_distance] = max_distance
target = (target/max_distance)*255
target = target.astype(np.uint8)
sparse = sparses[i] # (shape: (h, w))
sparse[sparse > max_distance] = max_distance
sparse = (sparse/max_distance)*255
sparse = sparse.astype(np.uint8)
pred[pred > max_distance] = max_distance
pred = (pred/max_distance)*255
pred = pred.astype(np.uint8)
sparse_color = cv2.applyColorMap(sparse, cv2.COLORMAP_SUMMER)
sparse_color[sparse == 0] = 0
target_color = cv2.applyColorMap(target, cv2.COLORMAP_SUMMER)
target_color[target == 0] = 0
pred_color = cv2.applyColorMap(pred, cv2.COLORMAP_SUMMER)
max_interval_length = 75.0 # (corresponds to the maximum length of a 95% conf interval)
max_sigma = max_interval_length/(2.0*1.96)
sigma_alea[sigma_alea > max_sigma] = max_sigma
sigma_alea = (sigma_alea/max_sigma)*255
sigma_alea = sigma_alea.astype(np.uint8)
sigma_alea_color = cv2.applyColorMap(sigma_alea, cv2.COLORMAP_HOT)
x_vis = x.copy()
x = preprocessing_fn(x)
y_pred = model.predict(np.expand_dims(x, axis=0))
y1_pred = np.argmax(y_pred[1])
y2_pred = np.argmax(np.squeeze(y_pred[0]), axis=-1)
y2_true = get_image(dataset_dir + 'drivable_maps/labels/val/' + name.split('.jpg')[0] + "_drivable_id.png")
n_background += np.count_nonzero(y2_true==0)
n_direct += np.count_nonzero(y2_true==1)
n_alternative += np.count_nonzero(y2_true==2)
if save_results and i%save_num == 0:
# PREDICTION
vis_pred = cv2.addWeighted(x_vis,1,cv2.applyColorMap(y2_pred.astype(np.uint8)*127,cv2.COLORMAP_OCEAN),1,0)
text = 'Prediction: {}'.format(tod_classes[y1_pred])
(text_width, text_height) = cv2.getTextSize(text, font, fontScale=fontScale, thickness=1)[0]
box_coords = ((textPosition[0] - 10, textPosition[1] + 10), (textPosition[0] + text_width + 5, textPosition[1] - text_height - 10))
cv2.rectangle(vis_pred, box_coords[0], box_coords[1], (0,0,0), cv2.FILLED)
cv2.putText(vis_pred, text, textPosition, font, fontScale, fontColor, lineType)
if visualize:
plt.imshow(vis_pred)
# GROUND TRUTH
vis_true = cv2.addWeighted(x_vis,1,cv2.applyColorMap(y2_true.astype(np.uint8)*127,cv2.COLORMAP_OCEAN),1,0)
text = 'Ground Truth: {}'.format(tod_classes[y1_true])
size_upsample = (origin_im_cropped.shape[1], origin_im_cropped.shape[0]
) # (w, h)
x = x.astype('float32') / 255.0
for j in inv_mapping.keys():
print('\t' + inv_mapping[j])
output, grads_val = sess.run(
[gradCam.target, grads[j]],
feed_dict={image_tensor: np.expand_dims(x, axis=0)})
cam = generate_cam(output[0], grads_val[0], size_upsample)
# Overlay cam on image
cam = np.uint8(255 * cam3)
cam = cv2.applyColorMap(cam3, cv2.COLORMAP_JET)
new_im = cam * 0.3 + origin_im_cropped * 0.5
if (next_image_ground_truth == next_image_prediction):
next_save_path = os.path.join(
args.outdir, 'correct',
'ground_truth_' + next_image_ground_truth,
'heatmap_' + inv_mapping[j],
next_image_name).replace('\\', '/')
else:
next_save_path = os.path.join(
args.outdir, 'incorrect',
'ground_truth_' + next_image_ground_truth,
'pred_' + next_image_prediction,
'heatmap_' + inv_mapping[j],
next_image_name).replace('\\', '/')
while True:
# Get capture
pyK4A.device_get_capture()
# Get the depth image from the capture
depth_image_handle = pyK4A.capture_get_depth_image()
# Check the image has been read correctly
if depth_image_handle:
# Read and convert the image data to numpy array:
depth_image = pyK4A.image_convert_to_numpy(depth_image_handle)
depth_color_image = cv2.convertScaleAbs(
depth_image, alpha=0.05
) #alpha is fitted by visual comparison with Azure k4aviewer results
depth_color_image = cv2.applyColorMap(depth_color_image,
cv2.COLORMAP_JET)
cv2.namedWindow('Colorized Depth Image', cv2.WINDOW_NORMAL)
cv2.imshow('Colorized Depth Image', depth_color_image)
k = cv2.waitKey(1)
# Release the image
pyK4A.image_release(depth_image_handle)
pyK4A.capture_release()
if k == 27: # Esc key to stop
break
pyK4A.device_stop_cameras()
pyK4A.device_close()
# img = Image.open(img_path).convert('RGB')
img = torch.load(img_path)
# img_inpug = preprocess(img).unsqueeze(0).cuda()
img_inpug = img.unsqueeze(0).cuda()
img_inpug.requires_grad_(True)
ret, mask, pred = grad_cam(img_inpug, class_index)
img_np = img.data.cpu().numpy() / 255
img_np = img_np.transpose((1, 2, 0))
heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)[:, :, ::-1]
heatmap = np.float32(heatmap) / 255
cam = 0.5 * np.float32(img_np) + 0.5 * heatmap
cam = cam / np.max(cam)
# print(cam.shape)
Image.fromarray(np.uint8(255 * cam)).save('cam.jpg')
# print(model)
max_num = 0
max_i, max_j = 0, 0
for i in range(mask.shape[0]):
for j in range(mask.shape[1]):
if mask[i][j] > max_num:
max_num = mask[i][j]
max_i, max_j = i, j
def main(video_dir):
# load videos
filenames = sorted(glob.glob(os.path.join(video_dir, "img/*.jpg")),
key=lambda x: int(os.path.basename(x).split('.')[0]))
# frames = [cv2.cvtColor(cv2.imread(filename), cv2.COLOR_BGR2RGB) for filename in filenames]
frames = [np.array(Image.open(filename)) for filename in filenames]
height, width = frames[0].shape[:2]
if len(frames[0].shape) == 3:
is_color = True
else:
is_color = False
frames = [frame[:, :, np.newaxis] for frame in frames]
gt_bboxes = pd.read_csv(os.path.join(video_dir, "groundtruth_rect.txt"), sep='\t|,| ',
header=None, names=['xmin', 'ymin', 'width', 'height'],
engine='python')
title = video_dir.split('/')[-1]
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
# img_writer = cv2.VideoWriter(os.path.join('./videos', title+'.avi'),
# fourcc, 25, (width, height))
# starting tracking
tracker = ECOTracker(is_color)
vis = True
for idx, frame in enumerate(frames):
if idx == 0:
bbox = gt_bboxes.iloc[0].values
tracker.init(frame, bbox)
bbox = (bbox[0]-1, bbox[1]-1,
bbox[0]+bbox[2]-1, bbox[1]+bbox[3]-1)
elif idx < len(frames) - 1:
bbox = tracker.update(frame, True, vis)
else: # last frame
bbox = tracker.update(frame, False, vis)
# bbox xmin ymin xmax ymax
frame = frame.squeeze()
if len(frame.shape) == 3:
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
else:
frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
frame = cv2.rectangle(frame,
(int(bbox[0]), int(bbox[1])),
(int(bbox[2]), int(bbox[3])),
(0, 255, 255),
1)
gt_bbox = gt_bboxes.iloc[idx].values
gt_bbox = (gt_bbox[0], gt_bbox[1],
gt_bbox[0]+gt_bbox[2], gt_bbox[1]+gt_bbox[3])
frame = frame.squeeze()
frame = cv2.rectangle(frame,
(int(gt_bbox[0]-1), int(gt_bbox[1]-1)), # 0-index
(int(gt_bbox[2]-1), int(gt_bbox[3]-1)),
(0, 255, 0),
1)
if vis and idx > 0:
score = tracker.score
size = tuple(tracker.crop_size.astype(np.int32))
score = cv2.resize(score, size)
score -= score.min()
score /= score.max()
score = (score * 255).astype(np.uint8)
# score = 255 - score
score = cv2.applyColorMap(score, cv2.COLORMAP_JET)
pos = tracker._pos
pos = (int(pos[0]), int(pos[1]))
xmin = pos[1] - size[1]//2
xmax = pos[1] + size[1]//2 + size[1] % 2
ymin = pos[0] - size[0] // 2
ymax = pos[0] + size[0] // 2 + size[0] % 2
left = abs(xmin) if xmin < 0 else 0
xmin = 0 if xmin < 0 else xmin
right = width - xmax
xmax = width if right < 0 else xmax
right = size[1] + right if right < 0 else size[1]
top = abs(ymin) if ymin < 0 else 0
ymin = 0 if ymin < 0 else ymin
down = height - ymax
ymax = height if down < 0 else ymax
down = size[0] + down if down < 0 else size[0]
score = score[top:down, left:right]
crop_img = frame[ymin:ymax, xmin:xmax]
# if crop_img.shape != score.shape:
# print(left, right, top, down)
# print(xmin, ymin, xmax, ymax)
score_map = cv2.addWeighted(crop_img, 0.6, score, 0.4, 0)
frame[ymin:ymax, xmin:xmax] = score_map
frame = cv2.putText(frame, str(idx), (5, 20), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 255, 0), 1)
# img_writer.write(frame)
cv2.imshow(title, frame)
cv2.waitKey(1)
def startVideo(self):
self._running = True
config = camera_config_path
if config is not None:
with open(config) as json_file:
config = json.load(json_file)
output_folder = config['path_dataset']
path_depth = join(output_folder, "depth")
path_color = join(output_folder, "color")
self.make_clean_folder(output_folder)
self.make_clean_folder(path_depth)
self.make_clean_folder(path_color)
# Create a pipeline
pipeline = rs.pipeline()
# Create a config and configure the pipeline to stream
# different resolutions of color and depth streams
config = rs.config()
config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16,
self.frame_rate)
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8,
self.frame_rate)
# Start streaming
profile = pipeline.start(config)
depth_sensor = profile.get_device().first_depth_sensor()
# Using preset HighAccuracy for recording
depth_sensor.set_option(rs.option.visual_preset, Preset.HighAccuracy)
# Getting the depth sensor's depth scale (see rs-align example for explanation)
depth_scale = depth_sensor.get_depth_scale()
# We will not display the background of objects more than
# clipping_distance_in_meters meters away
clipping_distance_in_meters = 3 # 3 meter
clipping_distance = clipping_distance_in_meters / depth_scale
# Create an align object
# rs.align allows us to perform alignment of depth frames to others frames
# The "align_to" is the stream type to which we plan to align depth frames.
align_to = rs.stream.color
align = rs.align(align_to)
# Streaming loop
frame_count = 0
try:
while self._running:
QCoreApplication.processEvents()
# Get frameset of color and depth
frames = pipeline.wait_for_frames()
# Align the depth frame to color frame
aligned_frames = align.process(frames)
# Get aligned frames
aligned_depth_frame = aligned_frames.get_depth_frame()
color_frame = aligned_frames.get_color_frame()
# Validate that both frames are valid
if not aligned_depth_frame or not color_frame:
continue
depth_image = np.asanyarray(aligned_depth_frame.get_data())
color_image = np.asanyarray(color_frame.get_data())
if frame_count == 0:
self.save_intrinsic_as_json(
join(output_folder, "camera_intrinsic.json"),
color_frame)
cv2.imwrite("%s/%06d.png" % \
(path_depth, frame_count), depth_image)
cv2.imwrite("%s/%06d.jpg" % \
(path_color, frame_count), color_image)
print("Saved color + depth image %06d" % frame_count)
frame_count += 1
# Remove background - Set pixels further than clipping_distance to grey
grey_color = 153
# depth image is 1 channel, color is 3 channels
depth_image_3d = np.dstack(
(depth_image, depth_image, depth_image))
bg_removed = np.where((depth_image_3d > clipping_distance) | \
(depth_image_3d <= 0), grey_color, color_image)
# Render images
depth_colormap = cv2.applyColorMap(
cv2.convertScaleAbs(depth_image, alpha=0.09),
cv2.COLORMAP_JET)
images = np.hstack((bg_removed, depth_colormap))
color_swapped_image = cv2.cvtColor(images, cv2.COLOR_BGR2RGB)
height, width, _ = images.shape
qt_image = QImage(color_swapped_image, width, height,
color_swapped_image.strides[0],
QImage.Format_RGB888).scaledToWidth(720)
# cv2.namedWindow('Recorder Realsense', cv2.WINDOW_AUTOSIZE)
# cv2.imshow('Recorder Realsense', images)
key = cv2.waitKey(1)
# if 'esc' button pressed, escape loop and exit program
if key == 27:
cv2.destroyAllWindows()
break
self.VideoSignal.emit(qt_image)
finally:
pipeline.stop()
def show_cam_on_image(img, mask, image_name=""):
heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
cam = heatmap + np.float32(img)
cam = cam / np.max(cam)
cv2.imwrite("cam.jpg", np.uint8(255 * cam))
# Create the IR image
ir_map = ir_map[0: int(ir_map.shape[0] / 2), :]
ir_map = distance_scale_ir * ir_map
ir_map = np.uint8(ir_map)
ir_map = cv.cvtColor(ir_map, cv.COLOR_GRAY2RGB)
# Show IR image
vis_ir.add_geometry(transform_image(ir_map))
vis_ir.poll_events()
# Create the Depth image
new_shape = (int(depth_map.shape[0] / 2), depth_map.shape[1])
depth16bits_map = depth_map = np.resize(depth_map, new_shape)
depth_map = distance_scale * depth_map
depth_map = np.uint8(depth_map)
depth_map = cv.applyColorMap(depth_map, cv.COLORMAP_RAINBOW)
# Show depth image
vis_depth.add_geometry(transform_image(depth_map))
vis_depth.poll_events()
# Create color image
img_color = cv.addWeighted(ir_map, 0.4, depth_map, 0.6, 0)
color_image = o3d.geometry.Image(img_color)
depth16bits_image = o3d.geometry.Image(depth16bits_map)
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(color_image, depth16bits_image, 1000.0, 3.0, False)
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, cameraIntrinsics)
# Flip it, otherwise the point cloud will be upside down
def cvt2HeatmapImg(img):
img = (np.clip(img, 0, 1) * 255).astype(np.uint8)
img = cv2.applyColorMap(img, cv2.COLORMAP_JET)
return img
def main():
global new_image
rs_img = rs_process()
rospy.init_node('hand_tracking', anonymous=True)
rospy.loginfo("Hand Detection Start!")
#Marker Publisher Initialize
pub = rospy.Publisher('/hand_marker', Marker)
hand_mark = MarkerGenerator()
hand_mark.type = Marker.SPHERE_LIST
hand_mark.scale = [.04] * 3
hand_mark.frame_id = '/camera_color_optical_frame'
# hand_mark.frame_id = 'iiwa_link_0'
#hand detect args
parser = argparse.ArgumentParser()
parser.add_argument('-sth',
'--scorethreshold',
dest='score_thresh',
type=float,
default=0.2,
help='Score threshold for displaying bounding boxes')
parser.add_argument('-fps',
'--fps',
dest='fps',
type=int,
default=1,
help='Show FPS on detection/display visualization')
parser.add_argument('-src',
'--source',
dest='video_source',
default=0,
help='Device index of the camera.')
parser.add_argument('-wd',
'--width',
dest='width',
type=int,
default=640,
help='Width of the frames in the video stream.')
parser.add_argument('-ht',
'--height',
dest='height',
type=int,
default=360,
help='Height of the frames in the video stream.')
parser.add_argument(
'-ds',
'--display',
dest='display',
type=int,
default=0,
help='Display the detected images using OpenCV. This reduces FPS')
parser.add_argument('-num-w',
'--num-workers',
dest='num_workers',
type=int,
default=4,
help='Number of workers.')
parser.add_argument('-q-size',
'--queue-size',
dest='queue_size',
type=int,
default=5,
help='Size of the queue.')
args = parser.parse_args()
num_hands_detect = 2
im_width, im_height = (args.width, args.height)
#time for fps calculation
start_time = datetime.datetime.now()
num_frames = 0
while not rospy.is_shutdown():
#get rgb,depth frames for synchronized frames
if not new_image:
continue
im_rgb = rs_image_rgb
im_depth = rs_image_depth
new_image = False
#add check
depth_map = cv2.applyColorMap(
cv2.convertScaleAbs(rs_image_depth, alpha=0.03), cv2.COLORMAP_JET)
cv2.imshow("Depth Image", depth_map)
cv2.imshow("Color Image", rs_image_rgb)
try:
image_np = im_rgb #cv2.cvtColor(im_rgb, cv2.COLOR_BGR2RGB)
except:
print("Error converting to RGB")
# Remove background - Set pixels further than clipping_distance to grey
grey_color = 100
depth_image_3d = np.dstack(
(im_depth, im_depth,
im_depth)) #depth image is 1 channel, color is 3 channels
bg_removed = np.where(
(depth_image_3d > clipping_distance) | (depth_image_3d <= 0),
grey_color, image_np)
#
img_hsv = cv2.cvtColor(bg_removed, cv2.COLOR_BGR2HSV)
# lower_red = np.array([120,150,50])
# upper_red = np.array([255,255,180])
## Gen lower mask (0-5) and upper mask (175-180) of RED
mask1 = cv2.inRange(img_hsv, (0, 50, 20), (5, 255, 255))
mask2 = cv2.inRange(img_hsv, (175, 50, 20), (180, 255, 255))
mask3 = cv2.inRange(img_hsv, (120, 150, 50), (255, 255, 180))
## Merge the mask and crop the red regions
mask = cv2.bitwise_or(mask1, mask3)
croped = cv2.bitwise_and(bg_removed, bg_removed, mask=mask)
# croped = cv2.cvtColor(croped, cv2.COLOR_BGR2GRAY)
img_bw = 255 * (cv2.cvtColor(croped, cv2.COLOR_BGR2GRAY) >
10).astype('uint8')
se1 = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
se2 = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
mask4 = cv2.morphologyEx(img_bw, cv2.MORPH_CLOSE, se1)
mask4 = cv2.morphologyEx(mask4, cv2.MORPH_OPEN, se2)
mask5 = np.dstack([mask4, mask4, mask4]) / 255
gray_croped = croped * mask5
# print(gray_croped.shape)
box_index = np.where(gray_croped != 0)
#check if the array is empty or not
if (np.array(box_index).size == 0):
continue
# print(box_index)
i_min = np.amin(box_index[0])
i_max = np.amax(box_index[0])
j_min = np.amin(box_index[1])
j_max = np.amax(box_index[1])
i_centor = int(i_min + i_max) / 2
j_centor = int(j_min + j_max) / 2
box_img = croped[i_min:i_max, j_min:j_max]
print(i_centor, j_centor)
## Display
cv2.imshow("mask", mask)
# cv2.namedWindow('RealSense', cv2.WINDOW_AUTOSIZE)
#check if the size is valied or not
if (box_img.shape[0] == 0 or box_img.shape[0] == 0
or box_img.shape[0] == 0):
continue
top = j_min
bottom = j_max
left = i_min
right = i_max
align_hand = im_rgb[int(top):int(bottom), int(left):int(right)]
align_depth = depth_map[int(top):int(bottom), int(left):int(right)]
align_hand_detect = np.hstack((align_hand, align_depth))
if (align_hand_detect.shape[0] > 0 and align_hand_detect.shape[1] > 0):
cv2.namedWindow('align hand', cv2.WINDOW_AUTOSIZE)
cv2.imshow('align hand', align_hand_detect)
# cv2.namedWindow("Depth Image", cv2.WINDOW_NORMAL)
# cv2.imshow("Depth Image", im_depth)
# cv2.namedWindow("Color Image", cv2.WINDOW_NORMAL)
# cv2.imshow("Color Image", im_rgb)
cv2.circle(bg_removed, (j_centor, i_centor),
radius=7,
color=(0, 0, 255),
thickness=-1)
cv2.imshow("bg_removed", bg_removed)
print('crop size: ', box_img.shape)
cv2.imshow("croped", box_img)
depth_pixel = [i_centor, j_centor]
depth_point = rs.rs2_deproject_pixel_to_point(
depth_intrin, depth_pixel,
im_depth[depth_pixel[0], depth_pixel[1]] * depth_scale)
print depth_point
hand_mark.counter = 0
t = rospy.get_time()
hand_mark.color = [1, 0, 0, 1]
m = hand_mark.marker(points=[(depth_point[1], depth_point[0],
depth_point[2])])
# rospy.loginfo(m)
pub.publish(m)
# rate.sleep()
if cv2.waitKey(15) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
continue
try:
image_np = im_rgb #cv2.cvtColor(im_rgb, cv2.COLOR_BGR2RGB)
except:
print("Error converting to RGB")
# cv2.imshow("source image np", image_np)
# actual hand detection
boxes, scores = detector_utils.detect_objects(image_np,
detection_graph, sess)
# draw bounding boxes
detector_utils.draw_box_on_image(num_hands_detect, args.score_thresh,
scores, boxes, im_width, im_height,
image_np)
if (scores[0] > args.score_thresh):
(left, right, top,
bottom) = (boxes[0][1] * im_width, boxes[0][3] * im_width,
boxes[0][0] * im_height, boxes[0][2] * im_height)
p1 = (int(left), int(top))
p2 = (int(right), int(bottom))
# print p1,p2,int(left),int(top),int(right),int(bottom)
image_hand = image_np[int(top):int(bottom), int(left):int(right)]
# cv2.namedWindow("hand", cv2.WINDOW_NORMAL)
# cv2.imshow('hand', cv2.cvtColor(image_hand, cv2.COLOR_RGB2BGR))
align_hand = im_rgb[int(top):int(bottom), int(left):int(right)]
align_depth = depth_map[int(top):int(bottom), int(left):int(right)]
align_hand_detect = np.hstack((align_hand, align_depth))
# cv2.namedWindow('align hand', cv2.WINDOW_AUTOSIZE)
# cv2.imshow('align hand', align_hand_detect)
#skin filtering
converted = cv2.cvtColor(align_hand, cv2.COLOR_BGR2HSV)
skinMask = cv2.inRange(converted, lower, upper)
# apply a series of erosions and dilations to the mask
# using an elliptical kernel
# kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11, 11))
# skinMask = cv2.erode(skinMask, kernel, iterations = 2)
# skinMask = cv2.dilate(skinMask, kernel, iterations = 2)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
skinMask = cv2.erode(skinMask, kernel, iterations=1)
skinMask = cv2.dilate(skinMask, kernel, iterations=1)
# blur the mask to help remove noise, then apply the
# mask to the frame
skinMask = cv2.GaussianBlur(skinMask, (3, 3), 0)
skin = cv2.bitwise_and(align_hand, align_hand, mask=skinMask)
depth_pixel = [(int(left) + int(right)) / 2,
(int(top) + int(bottom)) / 2]
depth_point = rs.rs2_deproject_pixel_to_point(
depth_intrin, depth_pixel,
im_depth[depth_pixel[1], depth_pixel[0]] * depth_scale)
print depth_point
hand_mark.counter = 0
t = rospy.get_time()
hand_mark.color = [1, 0, 0, 1]
m = hand_mark.marker(points=[(depth_point[0], depth_point[1],
depth_point[2])])
# rospy.loginfo(m)
pub.publish(m)
# rate.sleep()
# show the skin in the image along with the mask
# cv2.imshow("images", np.hstack([align_hand, skin]))
#end skin
# Calculate Frames per second (FPS)
num_frames += 1
elapsed_time = (datetime.datetime.now() - start_time).total_seconds()
fps = num_frames / elapsed_time
#display window
if (args.display > 0):
# Display FPS on frame
if (args.fps > 0):
detector_utils.draw_fps_on_image("FPS : " + str(float(fps)),
image_np)
cv2.imshow('Single Threaded Detection',
cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR))
else:
print("frames processed: ", num_frames, "elapsed time: ",
elapsed_time, "fps: ", str(int(fps)))
if cv2.waitKey(10) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
def main():
capture = cv2.VideoCapture('input.mp4')
background_subtractor = cv2.createBackgroundSubtractorMOG2()
length = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
bar = Bar('Processing Frame', max=length)
first_iteration_indicator = 1
for i in range(0, length):
ret, frame = capture.read()
#If first frame
if first_iteration_indicator == 1:
first_frame = copy.deepcopy(frame)
height, width = frame.shape[:2]
accum_image = np.zeros((height, width), np.uint8)
first_iteration_indicator = 0
else:
filter = background_subtractor.apply(frame)
cv2.imwrite('./frame.jpg', frame)
cv2.imwrite('./diff-bkgnd-frame.jpg', filter)
threshold = 2
maxValue = 2
ret, th1 = cv2.threshold(filter, threshold, maxValue,
cv2.THRESH_BINARY)
#add to the accumulated image
accum_image = cv2.add(accum_image, th1)
cv2.imwrite('./mask.jpg', accum_image)
color_image_video = cv2.applyColorMap(accum_image,
cv2.COLORMAP_SUMMER)
video_frame = cv2.addWeighted(frame, 0.7, color_image_video, 0.7,
0)
name = "./frames/frame%d.jpg" % i
print(name)
cv2.imwrite(name, video_frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
bar.next()
bar.finish()
make_video('./frames/', './output.avi')
color_image = cv2.applyColorMap(accum_image, cv2.COLORMAP_HOT)
result_overlay = cv2.addWeighted(first_frame, 0.7, color_image, 0.7, 0)
#save the final heatmap
cv2.imwrite('diff-overlay.jpg', result_overlay)
#cleanup
capture.release()
cv2.destroyAllWindows()
def log_heat_map(category_labels, x_val, image_size, model, model_source,
result_dir, random_state, initial_epoch):
for l in category_labels:
hm_count = 0
for i in range(len(x_val)):
if l in x_val[i]:
x = tf.keras.preprocessing.image.load_img(x_val[i], target_size=(image_size, image_size))
x = tf.keras.preprocessing.image.img_to_array(x)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x, model_source)
with tf.GradientTape() as tape:
last_conv_layer = model.get_layer(find_target_layer(model))
iterate = tf.keras.models.Model([model.inputs], [model.output, last_conv_layer.output])
model_out, conv_out = iterate(x)
class_out = model_out[:, np.argmax(model_out[0])]
grads = tape.gradient(class_out, conv_out)
pooled_grads = K.mean(grads, axis=(0, 1, 2))
hm_count += 1
heatmap = tf.multiply(pooled_grads, conv_out)
heatmap = tf.reduce_mean(heatmap, axis=-1)
heatmap = tf.reduce_mean(heatmap, axis=0)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
img = cv2.imread(x_val[i])
INTENSITY = 0.5
heatmap1 = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap1 = cv2.applyColorMap(np.uint8(255 * heatmap1), cv2.COLORMAP_VIRIDIS)
hm_image = heatmap1 * INTENSITY + img
cv2.imwrite(os.path.join(
result_dir,
'{}_hm_{}_{}_{}_{}.png'.format(random_state, initial_epoch, l, i, img.shape[1])),
hm_image)
# cv2.imwrite(os.path.join(
# result_dir,
# '{}_hm_{}_{}_{}_{}_h.png'.format(random_state, initial_epoch, l, i, img.shape[1])),
# heatmap1)
cv2.imwrite(os.path.join(
result_dir,
'{}_hm_{}_{}_{}_{}_i.png'.format(random_state, initial_epoch, l, i, img.shape[1])),
img)
original_image_file = x_val[i].replace('/{}/'.format(image_size), '/origin/')
img = cv2.cvtColor(load_crop_image(original_image_file, 1024), cv2.COLOR_RGB2BGR)
INTENSITY = 0.5
heatmap1 = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap1 = cv2.applyColorMap(np.uint8(255 * heatmap1), cv2.COLORMAP_VIRIDIS)
hm_image = heatmap1 * INTENSITY + img
cv2.imwrite(os.path.join(
result_dir,
'{}_hm_{}_{}_{}_{}.png'.format(random_state, initial_epoch, l, i, img.shape[1])),
hm_image)
# cv2.imwrite(os.path.join(
# result_dir,
# '{}_hm_{}_{}_{}_{}_h.png'.format(random_state, initial_epoch, l, i, img.shape[1])),
# heatmap1)
cv2.imwrite(os.path.join(
result_dir,
'{}_hm_{}_{}_{}_{}_i.png'.format(random_state, initial_epoch, l, i, img.shape[1])),
img)
# cv2.imwrite(os.path.join(
# result_dir,
# '{}_hm_{}_{}_{}_o.png'.format(random_state, initial_epoch, l, i)),
# img)
if hm_count >= 100:
break
def YOLO():
global metaMain, netMain, altNames
configPath = "./cfg/yolov3-spp.cfg"
weightPath = "./yolov3-spp.weights"
metaPath = "./cfg/coco.data"
if not os.path.exists(configPath):
raise ValueError("Invalid config path `" +
os.path.abspath(configPath) + "`")
if not os.path.exists(weightPath):
raise ValueError("Invalid weight path `" +
os.path.abspath(weightPath) + "`")
if not os.path.exists(metaPath):
raise ValueError("Invalid data file path `" +
os.path.abspath(metaPath) + "`")
if netMain is None:
netMain = darknet.load_net_custom(configPath.encode("ascii"),
weightPath.encode("ascii"), 0,
1) # batch size = 1
if metaMain is None:
metaMain = darknet.load_meta(metaPath.encode("ascii"))
if altNames is None:
try:
with open(metaPath) as metaFH:
metaContents = metaFH.read()
import re
match = re.search("names *= *(.*)$", metaContents,
re.IGNORECASE | re.MULTILINE)
if match:
result = match.group(1)
else:
result = None
try:
if os.path.exists(result):
with open(result) as namesFH:
namesList = namesFH.read().strip().split("\n")
altNames = [x.strip() for x in namesList]
except TypeError:
pass
except Exception:
pass
#cap = cv2.VideoCapture(0)
#create overlay for computing heatmap
overlay_empty = np.zeros((608, 608, 1), np.uint8)
cap = cv2.VideoCapture("video_test/01.avi") #IMPORTANT
cap.set(3, 1280)
cap.set(4, 720)
#print ("width", cap.get(3))
#print ("height", cap.get(4))
"""
out = cv2.VideoWriter(
"video_output/modded_2-5f_124_pytorch_MGN_thres00965_dthres0375_x30y160_age7000_cbox20-10w-5h-160.mp4", cv2.VideoWriter_fourcc(*"MP4V"), 20,
(darknet.network_width(netMain), darknet.network_height(netMain)))
print("Starting the YOLO loop...")
"""
out = cv2.VideoWriter(
"txt_output_n50_0960/01.avi",
cv2.VideoWriter_fourcc(*"MP4V"),
20, #IMPORTANT
(darknet.network_width(netMain), darknet.network_height(netMain)))
print("Starting the YOLO loop...")
# Create an image we reuse for each detect
darknet_image = darknet.make_image(darknet.network_width(netMain),
darknet.network_height(netMain), 3)
# ================= Definition of the parameters
max_cosine_distance = 0.096 #IMPORTANT PARAMETER
maxAge = 100 #IMPORTANT PARAMETER
nn_budget = None
nms_max_overlap = 1.0
# ================= Models location #IMPORTANT
#model_filename = 'model_data/mars-small128.pb' #Deep Cosine Metric model
model_filename = 'model_data/model_MGN.pt' #MGN model
# ========================= Init MGN feature extractor
extractor = Extractor(model_filename, use_cuda=True) #IMPORTANT
# ========================= Init Deep Cosine feature extractor
#encoder = gdet.create_box_encoder(model_filename,batch_size=1) #IMPORTANT
metric = nn_matching.NearestNeighborDistanceMetric("cosine",
max_cosine_distance,
nn_budget)
tracker = Tracker(metric, max_age=maxAge)
fps = 0.0
counts = 1
ids = dict()
b = set()
#fi = open("txt_output_n50_0960/01.txt","w")
while True:
prev_time = time.time()
ret, frame_read = cap.read()
if ret != True: break
if counts % 1 == 0:
print("========FRAME========: ", counts)
frame_rgb = cv2.cvtColor(frame_read, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb,
(darknet.network_width(netMain),
darknet.network_height(netMain)),
interpolation=cv2.INTER_LINEAR)
height, width, channels = frame_resized.shape
#print ("Shape of frame: ",frame_resized.shape)
darknet.copy_image_from_bytes(darknet_image,
frame_resized.tobytes())
boxs = darknet.detect_image(netMain,
metaMain,
darknet_image,
thresh=0.375)
boxes = []
for detection in boxs:
if detection[0].decode() != "person": continue
x, y, w, h = detection[2][0],\
detection[2][1],\
detection[2][2],\
detection[2][3]
#xmin = int(round(float(x) - (float(w) / 2)))
#ymin = int(round(float(y) - (float(h) / 2)))
xmin, ymin, xmax, ymax = convertBack(int(x), int(y), int(w),
int(h))
#========================= generate HEATMAP values ====================
x_heat = int(round((xmin + xmax) / 2))
overlay_empty[int(ymax) - 10:int(ymax) + 10,
x_heat - 10:x_heat + 10] += 5
#======================================================================
cv2.rectangle(frame_resized, (xmin, ymin), (xmax, ymax),
(255, 0, 255), 1)
if (xmin > 100 and ymin > 10) and (xmax < width - 5
and ymax < height - 120):
if (xmax - xmin) > 30 and (ymax - ymin) > 160:
boxes.append([xmin, ymin, w, h])
pt1 = (100, 10)
pt2 = (width - 5, height - 120)
cv2.rectangle(frame_resized, pt1, pt2, (0, 255, 0), 1)
# ==================== EXTRACT FEATURES ==================== #IMPORTANT
#features = encoder(frame_resized,boxes) #Deep Cosine Metric extractor
features = get_features(boxes, frame_resized, extractor, width,
height) #MGN extractor
# ============================================================
detections = [
Detection(bbox, 1.0, feature)
for bbox, feature in zip(boxes, features)
]
boxes = np.array([d.tlwh for d in detections])
scores = np.array([d.confidence for d in detections])
indices = preprocessing.non_max_suppression(
boxes, nms_max_overlap, scores)
detections = [detections[i] for i in indices]
#update tracker
tracker.predict()
tracker.update(detections)
# ====================== Fix the ID jumping problem ======================
if counts == 1:
count = 1
for track in tracker.tracks:
ids[track.track_id] = count
count += 1
else:
count = 0
for track in tracker.tracks:
if not track.is_confirmed() or track.time_since_update > 1:
continue
if track.track_id in ids: continue
if len(list(ids.values())) != 0:
count = max(list(ids.values()))
ids[track.track_id] = count + 1
#print (ids)
# =========================================================================
cv2.putText(frame_resized, "FrameID: " + str(counts), (350, 30), 0,
5e-3 * 150, (255, 255, 0), 2)
for track in tracker.tracks:
if not track.is_confirmed() or track.time_since_update > 1:
continue
#print ("=======ID at the frame=======: ", track.track_id)
bbox = track.to_tlbr()
#heat map
#print("x heat coordinate: ", x_heat)
cv2.rectangle(frame_resized, (int(bbox[0]), int(bbox[1])),
(int(bbox[2]), int(bbox[3])), (255, 255, 0), 2)
if track.track_id in ids:
cv2.putText(frame_resized, str(ids[track.track_id]),
(int(bbox[2]), int(bbox[3])), 0, 5e-3 * 250,
(255, 255, 0), 2)
if counts % 2 == 0:
fi.write(
str(counts) + " " + str(int(bbox[2])) + " " +
str(int(bbox[3])) + " " +
str(ids[track.track_id]) + " " + "\n")
b.add(track.track_id)
#if (np.allclose(overlay_empty[int(bbox[3])-5:int(bbox[3])+5,x_heat-5:x_heat+5,0],255)): continue
#print (b)
if len(list(ids.values())) != 0:
text = "Num people: {}".format(max(list(ids.values())))
cv2.putText(frame_resized, text, (1, 30), 0, 5e-3 * 200,
(255, 0, 0), 2)
overlay_empty[np.where((overlay_empty[:, :, 0] > 200))] = 230
"""
for det in detections:
bbox = det.to_tlbr()
cv2.rectangle(frame_resized,(int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])),(255,0,0), 2)
"""
# ============== BLENDING HEATMAP VALUES TO FRAME ==============
im_color = cv2.applyColorMap(overlay_empty, cv2.COLORMAP_JET)
im_color = cv2.blur(im_color, (10, 10))
#im_color[:,:,0]=0
image = cv2.cvtColor(frame_resized, cv2.COLOR_BGR2RGB)
#image = cvDrawBoxes(detections, frame_resized)
heat = cv2.addWeighted(image, 0.7, im_color, 0.3, 0)
# ===============================================================
#counts = counts + 1
out.write(heat)
print("fps: ", 1 / (time.time() - prev_time))
print("\n")
#fps = ( fps + (1./(time.time()-prev_time)) ) / 2
#print("fps= %f"%(fps))
#cv2.imshow('Demo', image)
#cv2.imshow('', heat)
#cv2.imshow('', cv2.resize(image,(1024,600)))
if cv2.waitKey(1) & 0xFF == ord('q'):
break
counts = counts + 1
fps = fps + (1 / (time.time() - prev_time))
print("average fps: ", fps / counts)
cap.release()
out.release()
cv2.destroyAllWindows()
def show_cam_on_image(img, mask, save_file):
heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
cam = heatmap + np.float32(img)
cam = cam / np.max(cam)
cv2.imwrite(save_file, np.uint8(255 * cam))
def show_cam_on_image(img, mask):
heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
cam = heatmap * 0.6 + np.float32(img)
cam = cam / np.max(cam)
return np.uint8(255 * cam)
# check_depth=cv2.imread(filename_depth,cv2.IMREAD_UNCHANGED)
# print((depth_image-check_depth).abs().sum())
if args.show:
# Remove background - Set pixels further than clipping_distance to grey
grey_color = 153
depth_image_3d = np.dstack(
(depth_image, depth_image,
depth_image)) #depth image is 1 channel, color is 3 channels
mask = (depth_image_3d < 0)
if clipping_distance:
mask = mask | (depth_image_3d > clipping_distance)
bg_removed = np.where(mask, grey_color, color_image)
# Render images
depth_colormap = cv2.applyColorMap(
cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)
images = np.hstack((bg_removed, depth_colormap))
cv2.namedWindow('Align Example', cv2.WINDOW_AUTOSIZE)
cv2.imshow('Align Example', images)
key = cv2.waitKey(1000)
# Press esc or 'q' to close the image window
if key & 0xFF == ord('q') or key == 27:
cv2.destroyAllWindows()
break
frame_i += 1
if args.num_frames and frame_i > args.num_frames:
break
if frame_i % 1e1 == 0:
# Reset progress bar
progress.completed = 0
