def predict_video(v_file_path): model_path = "deeplab_model.tar.gz" model = DeepLabModel(model_path) while True: vidObj = cv2.VideoCapture(v_file_path) count = 0 success = 1 while success: success, image = vidObj.read() count += 1 pil_im = Image.fromarray(image) resized_im, seg_map = model.run(pil_im) # color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) frame = np.array(pil_im) r = seg_image.shape[1] / frame.shape[1] dim = (int(frame.shape[0] * r), seg_image.shape[1])[::-1] resized = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA) resized = cv2.cvtColor(resized, cv2.COLOR_RGB2BGR) color_and_mask = np.hstack((resized, seg_image)) cv2.imshow('frame', color_and_mask) if cv2.waitKey(25) & 0xFF == ord('q'): cv2.destroyAllWindows() break
def testing(): download_path = os.path.join(model_dir, _TARBALL_NAME) model = DeepLab(download_path) cap = cv2.VideoCapture(0) final = np.zeros((1, 384, 1026, 3)) while True: ret, frame = cap.read() cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im) # model resized_im, seg_map = model.run(pil_im) # color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) frame = np.array(pil_im) r = seg_image.shape[1] / frame.shape[1] dim = (int(frame.shape[0] * r), seg_image.shape[1])[::-1] resized = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA) resized = cv2.cvtColor(resized, cv2.COLOR_RGB2BGR) color_and_mask = np.hstack((resized, seg_image)) cv2.imshow('frame', color_and_mask) if cv2.waitKey(25) & 0xFF == ord('q'): cap.release() cv2.destroyAllWindows() break
def vis_segmentation(image, seg_map): plt.figure(figsize=(15, 5)) grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1]) plt.subplot(grid_spec[0]) plt.imshow(image) plt.axis('off') plt.title('input image') plt.subplot(grid_spec[1]) seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) plt.imshow(seg_image) plt.axis('off') plt.title('segmentation map') plt.subplot(grid_spec[2]) plt.imshow(image) plt.imshow(seg_image, alpha=0.7) plt.axis('off') plt.title('segmentation overlay') unique_labels = np.unique(seg_map) ax = plt.subplot(grid_spec[3]) plt.imshow(FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest') ax.yaxis.tick_right() plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels]) plt.xticks([], []) ax.tick_params(width=0) plt.show()
def save_annotation(label, save_dir, filename, add_colormap=True, colormap_type=get_dataset_colormap.get_pascal_name()): """Saves the given label to image on disk. Args: label: The numpy array to be saved. The data will be converted to uint8 and saved as png image. save_dir: The directory to which the results will be saved. filename: The image filename. add_colormap: Add color map to the label or not. colormap_type: Colormap type for visualization. """ # Add colormap for visualizing the prediction. if add_colormap: colored_label = get_dataset_colormap.label_to_color_image( label, colormap_type) else: colored_label = label pil_image = img.fromarray(colored_label.astype(dtype=np.uint8)) with tf.gfile.Open('%s/%s.png' % (save_dir, filename), mode='w') as f: pil_image.save(f, 'PNG')
def run_threaded(self, framearr,rgbarr): while True: frame=framearr[0] start = time.time()#time processing of the frame # From cv2 to PIL pil_im = Image.fromarray(frame) # Run model resized_im, seg_map = model.run(pil_im) # Adjust color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) # Convert PIL image back to cv2 and resize r = seg_image.shape[1] / frame.shape[1] dim = (frame.shape[0], frame.shape[1])[::-1] #print(dim," ",frame.shape[0],"x",frame.shape[1]) seg_image = cv2.resize(seg_image, dim, interpolation = cv2.INTER_AREA) # Stack horizontally color frame and mask (r,g,b)=cv2.split(seg_image) ret2, thresh2 = cv2.threshold(r, 191, 255, cv2.THRESH_BINARY) ret3, thresh3 = cv2.threshold(g, 127, 255, cv2.THRESH_BINARY) ret4, thresh4 = cv2.threshold(b, 127, 255, cv2.THRESH_BINARY) rgb = cv2.bitwise_and(thresh2, thresh3, thresh4) rgb = cv2.merge((rgb,rgb,rgb)) rgbarr[0]=rgb #alpha = cv2.GaussianBlur(rgb, (7,7),0) end = time.time()#finish timer sys.stdout.write("\r%f fps" % (1/(end - start)))
def seperate_foreground_background(path_to_pb, path_to_image): graph = load_pb(path_to_pb) seg_map, image = fit_to_model(graph, path_to_image) seg_map = cv2.resize(seg_map, image.shape[:2][::-1], interpolation=cv2.INTER_NEAREST) logging.info('step 2') # Only keep the person class seg_map = np.where(seg_map == 15, seg_map, 0) # Output: RGB image resized from original input image, segmentation map of resized image # color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) seg_image_mono = (seg_image[:, :, 0] + seg_image[:, :, 1] + seg_image[:, :, 2]).astype(np.uint8) background = greyscales_2_color( (np.where(seg_image_mono == 0, image[:, :, 0], 0), np.where(seg_image_mono == 0, image[:, :, 1], 0), np.where(seg_image_mono == 0, image[:, :, 2], 0))) # io.imsave('background.jpg', img_as_ubyte(background)) foreground = greyscales_2_color( (np.where(seg_image_mono > 0, image[:, :, 0], 0), np.where(seg_image_mono > 0, image[:, :, 1], 0), np.where(seg_image_mono > 0, image[:, :, 2], 0))) return seg_image_mono, background, image
def run_once(self, framearr): frame=framearr[0] # From cv2 to PIL cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im) # Run model resized_im, seg_map = model.run(pil_im) # Adjust color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) # Convert PIL image back to cv2 and resize r = seg_image.shape[1] / frame.shape[1] dim = (frame.shape[0], frame.shape[1])[::-1] #print(dim," ",frame.shape[0],"x",frame.shape[1]) seg_image = cv2.resize(seg_image, dim, interpolation = cv2.INTER_AREA) # Stack horizontally color frame and mask (r,g,b)=cv2.split(seg_image) ret2, thresh2 = cv2.threshold(r, 191, 255, cv2.THRESH_BINARY) ret3, thresh3 = cv2.threshold(g, 127, 255, cv2.THRESH_BINARY) ret4, thresh4 = cv2.threshold(b, 127, 255, cv2.THRESH_BINARY) rgb = cv2.bitwise_and(thresh2, thresh3, thresh4) rgb = cv2.merge((rgb,rgb,rgb)) return rgb
def referenceBackground(cap): while (True): ret1, frame1 = cap.read() frame1RGB = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB) print('Capture reference image first') print('press p to capture background') cv2.imshow('background',frame1RGB) if cv2.waitKey(1) & 0xFF == ord('p'): cv2.destroyAllWindows() print('in p') pil_im = Image.fromarray(frame1RGB) # model model_time = time.time() resized_im, seg_map = model.run(pil_im) # color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) model_time = (time.time() - model_time) # cv2.imshow('background',seg_map) unique_labels,label_counts = np.unique(seg_map, return_counts=True) in_this_pic = LABEL_NAMES[unique_labels] grouped_label_counts = dict(zip(in_this_pic,label_counts)) # np.savetxt('reference.csv',grouped_label_counts,delimiter=',') # saving semantic data as json save_csv(grouped_label_counts,'reference') # loadcheck # open_json('reference.txt') cv2.imwrite('/Users/dt/Desktop/CodesTemp/ImmunityHealth/ImmunityHealthHackathon/SegmentationBasedhumandetection/referenceBackgroundSegmented.jpg', seg_image) break print('done capturing reference!') cv2.destroyAllWindows() return grouped_label_counts, model_time
def get_bimap(self, seg_map, image=None): seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) bimap = np.equal(seg_map, 0) + 0 unique_labels = np.unique(seg_map) if image: foregrounds = np.multiply(image, np.expand_dims(1 - bimap, axis=2)) return [foregrounds, bimap] return bimap
def main(): model = DeepLabModel(download_path) img, seg_map = run_demo_image(model, 'mb.jpg') seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) #cv2.imwrite('img/seg1.jpg',seg_image) #print(type(img), type(seg_image)) img = np.clip(img,0,255).astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) overlay = cv2.addWeighted(img, 0.3, seg_image, 0.7,0) cv2.imwrite('img/overlay1.jpg',overlay)
def vis_segmentation(image, seg_map, img_name): plt.figure() plt.subplot(221) plt.imshow(image) plt.axis('off') plt.title('input image') plt.subplot(222) seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) #print(seg_image.shape) #plt.imshow(seg_image(seg_image.shape[0],seg_image.shape[1])) #make all non zero pixel 1 bimap = np.equal(seg_map, 0) + 0 plt.imshow(bimap, cmap=plt.cm.binary) plt.axis('off') plt.title('segmentation map') plt.subplot(223) #plt.imshow(image) #plt.imshow(seg_image, alpha=0.7) backgrounds = np.multiply(image, np.expand_dims(bimap, axis=2)) plt.imshow(backgrounds) plt.axis('off') plt.title('background segmentation') unique_labels = np.unique(seg_map) ax = plt.subplot(224) foregrounds = np.multiply(image, np.expand_dims(1 - bimap, axis=2)) plt.imshow(foregrounds) plt.axis('off') plt.title('foreground segmentation') unique_labels = np.unique(seg_map) #bimap --> 1 denotes background(white) and 0 denotes foreground(black) #the result of seg map is inversed for better visualization '''plt.imshow( FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest') ax.yaxis.tick_right() plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels]) plt.xticks([], []) ax.tick_params(width=0) ''' misc.imsave('../../sample_img/{}_alpha.png'.format(img_name), 1 - bimap) misc.imsave('../../sample_img/{}_rgb.png'.format(img_name), image) plt.show()
def postureCorrection(reference_time): cap = cv2.VideoCapture(0) grouped_label_counts_reference = open_csv('reference') prev_time = time.time() while(cap.isOpened()): if (time.time() - prev_time) > reference_time: ret1, frame1 = cap.read() print("in here") frame1RGB = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(frame1RGB) resized_im, seg_map = model.run(pil_im) seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) unique_labels,label_counts = np.unique(seg_map, return_counts=True) in_this_pic = LABEL_NAMES[unique_labels] grouped_label_counts = dict(zip(in_this_pic,label_counts)) if grouped_label_counts['person'] > grouped_label_counts_reference['person']: print("Please adjust your posture and move a little backwards") elif grouped_label_counts['person'] < grouped_label_counts_reference['person']: print("Please adjust your posture and move a little foward") prev_time = time.time()
def main(): model = DeepLabModel(download_path) img, seg_map = run_demo_image(model, 'park.jpg') #print(seg_map) pickle.dump(seg_map, gzip.open('suzi.pkl','wb')) segMap = pickle.load(gzip.open('suzi.pkl','rb')) seg_image = get_dataset_colormap.label_to_color_image( segMap, get_dataset_colormap.get_pascal_name()).astype(np.uint8) #seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) cv2.imwrite('img/seg1.jpg',seg_image) #print(type(img), type(seg_image)) img = np.clip(img,0,255).astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) overlay = cv2.addWeighted(img, 0.3, seg_image, 0.7,0) cv2.imwrite('img/overlay1.jpg',overlay) blur = cv2.GaussianBlur(img, (9,9), 0) seg2gray = cv2.cvtColor(seg_image, cv2.COLOR_BGR2GRAY) ret, mask = cv2.threshold(seg2gray, 5, 255, cv2.THRESH_BINARY) cv2.imwrite('img/mask.jpg',mask) mask_inv = cv2.bitwise_not(mask) fg = cv2.bitwise_and(img, img, mask=mask) bg = cv2.bitwise_and(blur, blur, mask=mask_inv) graybg = cv2.bitwise_and(gray, gray, mask=mask_inv) dst = cv2.add(fg, bg) cv2.imwrite('img/blur.jpg',dst) graybg = cv2.cvtColor(graybg, cv2.COLOR_GRAY2BGR) dst = cv2.add(fg, graybg) cv2.imwrite('img/gray.jpg',dst)
def segmentation(self): if not self.imagemsg: return msg = deepcopy(self.imagemsg) try: img = self.cvbridge.imgmsg_to_cv2(msg,"bgr8") except CvBridgeError as e: rospy.logerr(e) return image = PIL.Image.fromarray(img[:, :, ::-1].copy()) width, height = image.size resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height) target_size = (int(resize_ratio * width), int(resize_ratio * height)) resized_image = image.convert('RGB').resize(target_size, PIL.Image.ANTIALIAS) batch_seg_map = self.sess.run( self.OUTPUT_TENSOR_NAME, feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]}) seg_map = batch_seg_map[0] seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) self.pub.publish(self.cvbridge.cv2_to_imgmsg(seg_image,"rgb8"))
def main(): model = DeepLabModel(download_path) #gif = imageio.mimread('park.gif') #print('gif num: ', len(gif)) gif =Image.open('park.gif') #img, seg_map = run_demo_image(model, 'park.jpg') #imgs = [cv2.cvtColor(img, cv2.COLOR_RGB2BGR) for img in gif] gif_blur=[] gif_gray=[] #for img in gif: nframes = 0 while gif: nframes += 1 #origin = Image.open(image_path) img, seg_map = model.run(gif) #print(seg_map) #pickle.dump(seg_map, gzip.open('suzi.pkl','wb')) #segMap = pickle.load(gzip.open('suzi.pkl','rb')) seg_image = get_dataset_colormap.label_to_color_image( #segMap, get_dataset_colormap.get_pascal_name()).astype(np.uint8) seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) #cv2.imwrite('img/seg1.jpg',seg_image) #print(type(img), type(seg_image)) img = np.clip(img,0,255).astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) #overlay = cv2.addWeighted(img, 0.3, seg_image, 0.7,0) #cv2.imwrite('img/overlay1.jpg',overlay) blur = cv2.GaussianBlur(img, (9,9), 0) seg2gray = cv2.cvtColor(seg_image, cv2.COLOR_BGR2GRAY) ret, mask = cv2.threshold(seg2gray, 5, 255, cv2.THRESH_BINARY) #cv2.imwrite('img/mask.jpg',mask) mask_inv = cv2.bitwise_not(mask) fg = cv2.bitwise_and(img, img, mask=mask) bg = cv2.bitwise_and(blur, blur, mask=mask_inv) graybg = cv2.bitwise_and(gray, gray, mask=mask_inv) dst = cv2.add(fg, bg) #cv2.imwrite('img/blur.jpg',dst) gif_blur.append(cv2.cvtColor(dst, cv2.COLOR_BGR2RGB)) graybg = cv2.cvtColor(graybg, cv2.COLOR_GRAY2BGR) dst_gray = cv2.add(fg, graybg) #cv2.imwrite('img/gray.jpg',dst_gray) gif_gray.append(cv2.cvtColor(dst_gray, cv2.COLOR_BGR2RGB)) try: gif.seek( nframes ) except EOFError: break; imageio.mimwrite('park_blur.gif', gif_blur, duration=0.01, loop=10) imageio.mimwrite('park_gray.gif', gif_gray, duration=0.03, loop=10)
def composeg(self, image, seg_map): seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) blend_img = Image.blend(image, Image.fromarray(seg_image), alpha=0.5) return blend_img
# Next line may need adjusting depending on webcam resolution final = np.zeros((1, 384, 1026, 3)) while True: ret, frame = cap.read() # From cv2 to PIL cv2_im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im) # Run model resized_im, seg_map = model.run(pil_im) # Adjust color of mask seg_image = get_dataset_colormap.label_to_color_image( seg_map, get_dataset_colormap.get_pascal_name()).astype(np.uint8) # Convert PIL image back to cv2 and resize frame = np.array(pil_im) r = seg_image.shape[1] / frame.shape[1] dim = (int(frame.shape[0] * r), seg_image.shape[1])[::-1] # resized = cv2.resize(frame, dim, interpolation = cv2.INTER_AREA) resized = cv2.resize(frame, dim) resized = cv2.cvtColor(resized, cv2.COLOR_RGB2BGR) # Stack horizontally color frame and mask color_and_mask = np.hstack((resized, seg_image)) cv2.imshow('frame', color_and_mask) if cv2.waitKey(25) & 0xFF == ord('q'): cap.release()