def predict_video(v_file_path):
model_path = "deeplab_model.tar.gz"
model = DeepLabModel(model_path)
while True:
vidObj = cv2.VideoCapture(v_file_path)
count = 0
success = 1
while success:
success, image = vidObj.read()
count += 1
pil_im = Image.fromarray(image)
resized_im, seg_map = model.run(pil_im)
# color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map,
get_dataset_colormap.get_pascal_name()).astype(np.uint8)
frame = np.array(pil_im)
r = seg_image.shape[1] / frame.shape[1]
dim = (int(frame.shape[0] * r), seg_image.shape[1])[::-1]
resized = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA)
resized = cv2.cvtColor(resized, cv2.COLOR_RGB2BGR)
color_and_mask = np.hstack((resized, seg_image))
cv2.imshow('frame', color_and_mask)
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
def testing():
download_path = os.path.join(model_dir, _TARBALL_NAME)
model = DeepLab(download_path)
cap = cv2.VideoCapture(0)
final = np.zeros((1, 384, 1026, 3))
while True:
ret, frame = cap.read()
cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im)
# model
resized_im, seg_map = model.run(pil_im)
# color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
frame = np.array(pil_im)
r = seg_image.shape[1] / frame.shape[1]
dim = (int(frame.shape[0] * r), seg_image.shape[1])[::-1]
resized = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA)
resized = cv2.cvtColor(resized, cv2.COLOR_RGB2BGR)
color_and_mask = np.hstack((resized, seg_image))
cv2.imshow('frame', color_and_mask)
if cv2.waitKey(25) & 0xFF == ord('q'):
cap.release()
cv2.destroyAllWindows()
break
def vis_segmentation(image, seg_map):
plt.figure(figsize=(15, 5))
grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])
plt.subplot(grid_spec[0])
plt.imshow(image)
plt.axis('off')
plt.title('input image')
plt.subplot(grid_spec[1])
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
plt.imshow(seg_image)
plt.axis('off')
plt.title('segmentation map')
plt.subplot(grid_spec[2])
plt.imshow(image)
plt.imshow(seg_image, alpha=0.7)
plt.axis('off')
plt.title('segmentation overlay')
unique_labels = np.unique(seg_map)
ax = plt.subplot(grid_spec[3])
plt.imshow(FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest')
ax.yaxis.tick_right()
plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])
plt.xticks([], [])
ax.tick_params(width=0)
plt.show()
def save_annotation(label,
save_dir,
filename,
add_colormap=True,
colormap_type=get_dataset_colormap.get_pascal_name()):
"""Saves the given label to image on disk.
Args:
label: The numpy array to be saved. The data will be converted
to uint8 and saved as png image.
save_dir: The directory to which the results will be saved.
filename: The image filename.
add_colormap: Add color map to the label or not.
colormap_type: Colormap type for visualization.
"""
# Add colormap for visualizing the prediction.
if add_colormap:
colored_label = get_dataset_colormap.label_to_color_image(
label, colormap_type)
else:
colored_label = label
pil_image = img.fromarray(colored_label.astype(dtype=np.uint8))
with tf.gfile.Open('%s/%s.png' % (save_dir, filename), mode='w') as f:
pil_image.save(f, 'PNG')
def run_threaded(self, framearr,rgbarr):
while True:
frame=framearr[0]
start = time.time()#time processing of the frame
# From cv2 to PIL
pil_im = Image.fromarray(frame)
# Run model
resized_im, seg_map = model.run(pil_im)
# Adjust color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
# Convert PIL image back to cv2 and resize
r = seg_image.shape[1] / frame.shape[1]
dim = (frame.shape[0], frame.shape[1])[::-1]
#print(dim," ",frame.shape[0],"x",frame.shape[1])
seg_image = cv2.resize(seg_image, dim, interpolation = cv2.INTER_AREA)
# Stack horizontally color frame and mask
(r,g,b)=cv2.split(seg_image)
ret2, thresh2 = cv2.threshold(r, 191, 255, cv2.THRESH_BINARY)
ret3, thresh3 = cv2.threshold(g, 127, 255, cv2.THRESH_BINARY)
ret4, thresh4 = cv2.threshold(b, 127, 255, cv2.THRESH_BINARY)
rgb = cv2.bitwise_and(thresh2, thresh3, thresh4)
rgb = cv2.merge((rgb,rgb,rgb))
rgbarr[0]=rgb
#alpha = cv2.GaussianBlur(rgb, (7,7),0)
end = time.time()#finish timer
sys.stdout.write("\r%f fps" % (1/(end - start)))
def seperate_foreground_background(path_to_pb, path_to_image):
graph = load_pb(path_to_pb)
seg_map, image = fit_to_model(graph, path_to_image)
seg_map = cv2.resize(seg_map,
image.shape[:2][::-1],
interpolation=cv2.INTER_NEAREST)
logging.info('step 2')
# Only keep the person class
seg_map = np.where(seg_map == 15, seg_map, 0)
# Output: RGB image resized from original input image, segmentation map of resized image
# color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
seg_image_mono = (seg_image[:, :, 0] + seg_image[:, :, 1] +
seg_image[:, :, 2]).astype(np.uint8)
background = greyscales_2_color(
(np.where(seg_image_mono == 0, image[:, :, 0],
0), np.where(seg_image_mono == 0, image[:, :, 1], 0),
np.where(seg_image_mono == 0, image[:, :, 2], 0)))
# io.imsave('background.jpg', img_as_ubyte(background))
foreground = greyscales_2_color(
(np.where(seg_image_mono > 0, image[:, :, 0],
0), np.where(seg_image_mono > 0, image[:, :, 1], 0),
np.where(seg_image_mono > 0, image[:, :, 2], 0)))
return seg_image_mono, background, image
def run_once(self, framearr):
frame=framearr[0]
# From cv2 to PIL
cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im)
# Run model
resized_im, seg_map = model.run(pil_im)
# Adjust color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
# Convert PIL image back to cv2 and resize
r = seg_image.shape[1] / frame.shape[1]
dim = (frame.shape[0], frame.shape[1])[::-1]
#print(dim," ",frame.shape[0],"x",frame.shape[1])
seg_image = cv2.resize(seg_image, dim, interpolation = cv2.INTER_AREA)
# Stack horizontally color frame and mask
(r,g,b)=cv2.split(seg_image)
ret2, thresh2 = cv2.threshold(r, 191, 255, cv2.THRESH_BINARY)
ret3, thresh3 = cv2.threshold(g, 127, 255, cv2.THRESH_BINARY)
ret4, thresh4 = cv2.threshold(b, 127, 255, cv2.THRESH_BINARY)
rgb = cv2.bitwise_and(thresh2, thresh3, thresh4)
rgb = cv2.merge((rgb,rgb,rgb))
return rgb
def referenceBackground(cap):
while (True):
ret1, frame1 = cap.read()
frame1RGB = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB)
print('Capture reference image first')
print('press p to capture background')
cv2.imshow('background',frame1RGB)
if cv2.waitKey(1) & 0xFF == ord('p'):
cv2.destroyAllWindows()
print('in p')
pil_im = Image.fromarray(frame1RGB)
# model
model_time = time.time()
resized_im, seg_map = model.run(pil_im)
# color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
model_time = (time.time() - model_time)
# cv2.imshow('background',seg_map)
unique_labels,label_counts = np.unique(seg_map, return_counts=True)
in_this_pic = LABEL_NAMES[unique_labels]
grouped_label_counts = dict(zip(in_this_pic,label_counts))
# np.savetxt('reference.csv',grouped_label_counts,delimiter=',')
# saving semantic data as json
save_csv(grouped_label_counts,'reference')
# loadcheck
# open_json('reference.txt')
cv2.imwrite('/Users/dt/Desktop/CodesTemp/ImmunityHealth/ImmunityHealthHackathon/SegmentationBasedhumandetection/referenceBackgroundSegmented.jpg', seg_image)
break
print('done capturing reference!')
cv2.destroyAllWindows()
return grouped_label_counts, model_time
def get_bimap(self, seg_map, image=None):
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
bimap = np.equal(seg_map, 0) + 0
unique_labels = np.unique(seg_map)
if image:
foregrounds = np.multiply(image, np.expand_dims(1 - bimap, axis=2))
return [foregrounds, bimap]
return bimap
def main():
model = DeepLabModel(download_path)
img, seg_map = run_demo_image(model, 'mb.jpg')
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
#cv2.imwrite('img/seg1.jpg',seg_image)
#print(type(img), type(seg_image))
img = np.clip(img,0,255).astype('uint8')
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
overlay = cv2.addWeighted(img, 0.3, seg_image, 0.7,0)
cv2.imwrite('img/overlay1.jpg',overlay)
def vis_segmentation(image, seg_map, img_name):
plt.figure()
plt.subplot(221)
plt.imshow(image)
plt.axis('off')
plt.title('input image')
plt.subplot(222)
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
#print(seg_image.shape)
#plt.imshow(seg_image(seg_image.shape[0],seg_image.shape[1]))
#make all non zero pixel 1
bimap = np.equal(seg_map, 0) + 0
plt.imshow(bimap, cmap=plt.cm.binary)
plt.axis('off')
plt.title('segmentation map')
plt.subplot(223)
#plt.imshow(image)
#plt.imshow(seg_image, alpha=0.7)
backgrounds = np.multiply(image, np.expand_dims(bimap, axis=2))
plt.imshow(backgrounds)
plt.axis('off')
plt.title('background segmentation')
unique_labels = np.unique(seg_map)
ax = plt.subplot(224)
foregrounds = np.multiply(image, np.expand_dims(1 - bimap, axis=2))
plt.imshow(foregrounds)
plt.axis('off')
plt.title('foreground segmentation')
unique_labels = np.unique(seg_map)
#bimap --> 1 denotes background(white) and 0 denotes foreground(black)
#the result of seg map is inversed for better visualization
'''plt.imshow(
FULL_COLOR_MAP[unique_labels].astype(np.uint8),
interpolation='nearest')
ax.yaxis.tick_right()
plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])
plt.xticks([], [])
ax.tick_params(width=0)
'''
misc.imsave('../../sample_img/{}_alpha.png'.format(img_name), 1 - bimap)
misc.imsave('../../sample_img/{}_rgb.png'.format(img_name), image)
plt.show()
def postureCorrection(reference_time):
cap = cv2.VideoCapture(0)
grouped_label_counts_reference = open_csv('reference')
prev_time = time.time()
while(cap.isOpened()):
if (time.time() - prev_time) > reference_time:
ret1, frame1 = cap.read()
print("in here")
frame1RGB = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(frame1RGB)
resized_im, seg_map = model.run(pil_im)
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
unique_labels,label_counts = np.unique(seg_map, return_counts=True)
in_this_pic = LABEL_NAMES[unique_labels]
grouped_label_counts = dict(zip(in_this_pic,label_counts))
if grouped_label_counts['person'] > grouped_label_counts_reference['person']:
print("Please adjust your posture and move a little backwards")
elif grouped_label_counts['person'] < grouped_label_counts_reference['person']:
print("Please adjust your posture and move a little foward")
prev_time = time.time()
def main():
model = DeepLabModel(download_path)
img, seg_map = run_demo_image(model, 'park.jpg')
#print(seg_map)
pickle.dump(seg_map, gzip.open('suzi.pkl','wb'))
segMap = pickle.load(gzip.open('suzi.pkl','rb'))
seg_image = get_dataset_colormap.label_to_color_image(
segMap, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
#seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
cv2.imwrite('img/seg1.jpg',seg_image)
#print(type(img), type(seg_image))
img = np.clip(img,0,255).astype('uint8')
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
overlay = cv2.addWeighted(img, 0.3, seg_image, 0.7,0)
cv2.imwrite('img/overlay1.jpg',overlay)
blur = cv2.GaussianBlur(img, (9,9), 0)
seg2gray = cv2.cvtColor(seg_image, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(seg2gray, 5, 255, cv2.THRESH_BINARY)
cv2.imwrite('img/mask.jpg',mask)
mask_inv = cv2.bitwise_not(mask)
fg = cv2.bitwise_and(img, img, mask=mask)
bg = cv2.bitwise_and(blur, blur, mask=mask_inv)
graybg = cv2.bitwise_and(gray, gray, mask=mask_inv)
dst = cv2.add(fg, bg)
cv2.imwrite('img/blur.jpg',dst)
graybg = cv2.cvtColor(graybg, cv2.COLOR_GRAY2BGR)
dst = cv2.add(fg, graybg)
cv2.imwrite('img/gray.jpg',dst)
def segmentation(self):
if not self.imagemsg:
return
msg = deepcopy(self.imagemsg)
try:
img = self.cvbridge.imgmsg_to_cv2(msg,"bgr8")
except CvBridgeError as e:
rospy.logerr(e)
return
image = PIL.Image.fromarray(img[:, :, ::-1].copy())
width, height = image.size
resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height)
target_size = (int(resize_ratio * width), int(resize_ratio * height))
resized_image = image.convert('RGB').resize(target_size, PIL.Image.ANTIALIAS)
batch_seg_map = self.sess.run(
self.OUTPUT_TENSOR_NAME,
feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})
seg_map = batch_seg_map[0]
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
self.pub.publish(self.cvbridge.cv2_to_imgmsg(seg_image,"rgb8"))
def main():
model = DeepLabModel(download_path)
#gif = imageio.mimread('park.gif')
#print('gif num: ', len(gif))
gif =Image.open('park.gif')
#img, seg_map = run_demo_image(model, 'park.jpg')
#imgs = [cv2.cvtColor(img, cv2.COLOR_RGB2BGR) for img in gif]
gif_blur=[]
gif_gray=[]
#for img in gif:
nframes = 0
while gif:
nframes += 1
#origin = Image.open(image_path)
img, seg_map = model.run(gif)
#print(seg_map)
#pickle.dump(seg_map, gzip.open('suzi.pkl','wb'))
#segMap = pickle.load(gzip.open('suzi.pkl','rb'))
seg_image = get_dataset_colormap.label_to_color_image(
#segMap, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
#cv2.imwrite('img/seg1.jpg',seg_image)
#print(type(img), type(seg_image))
img = np.clip(img,0,255).astype('uint8')
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#overlay = cv2.addWeighted(img, 0.3, seg_image, 0.7,0)
#cv2.imwrite('img/overlay1.jpg',overlay)
blur = cv2.GaussianBlur(img, (9,9), 0)
seg2gray = cv2.cvtColor(seg_image, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(seg2gray, 5, 255, cv2.THRESH_BINARY)
#cv2.imwrite('img/mask.jpg',mask)
mask_inv = cv2.bitwise_not(mask)
fg = cv2.bitwise_and(img, img, mask=mask)
bg = cv2.bitwise_and(blur, blur, mask=mask_inv)
graybg = cv2.bitwise_and(gray, gray, mask=mask_inv)
dst = cv2.add(fg, bg)
#cv2.imwrite('img/blur.jpg',dst)
gif_blur.append(cv2.cvtColor(dst, cv2.COLOR_BGR2RGB))
graybg = cv2.cvtColor(graybg, cv2.COLOR_GRAY2BGR)
dst_gray = cv2.add(fg, graybg)
#cv2.imwrite('img/gray.jpg',dst_gray)
gif_gray.append(cv2.cvtColor(dst_gray, cv2.COLOR_BGR2RGB))
try:
gif.seek( nframes )
except EOFError:
break;
imageio.mimwrite('park_blur.gif', gif_blur, duration=0.01, loop=10)
imageio.mimwrite('park_gray.gif', gif_gray, duration=0.03, loop=10)
def composeg(self, image, seg_map):
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
blend_img = Image.blend(image, Image.fromarray(seg_image), alpha=0.5)
return blend_img
# Next line may need adjusting depending on webcam resolution
final = np.zeros((1, 384, 1026, 3))
while True:
ret, frame = cap.read()
# From cv2 to PIL
cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im)
# Run model
resized_im, seg_map = model.run(pil_im)
# Adjust color of mask
seg_image = get_dataset_colormap.label_to_color_image(
seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8)
# Convert PIL image back to cv2 and resize
frame = np.array(pil_im)
r = seg_image.shape[1] / frame.shape[1]
dim = (int(frame.shape[0] * r), seg_image.shape[1])[::-1]
# resized = cv2.resize(frame, dim, interpolation = cv2.INTER_AREA)
resized = cv2.resize(frame, dim)
resized = cv2.cvtColor(resized, cv2.COLOR_RGB2BGR)
# Stack horizontally color frame and mask
color_and_mask = np.hstack((resized, seg_image))
cv2.imshow('frame', color_and_mask)
if cv2.waitKey(25) & 0xFF == ord('q'):
cap.release()
