class RendererV3(object): def __init__(self, data_dir, max_time=None): self.text_renderer = tu.RenderFont(data_dir) self.colorizer = Colorize(data_dir) self.min_char_height = 8 # px self.min_asp_ratio = 0.4 # self.max_text_regions = 7 self.max_time = max_time def filter_regions(self, regions, filt): """ filt : boolean list of regions to keep. """ idx = np.arange(len(filt))[filt] for k in regions.keys(): regions[k] = [regions[k][i] for i in idx] return regions def filter_for_placement(self, xyz, seg, regions): filt = np.zeros(len(regions['label'])).astype('bool') masks, Hs, Hinvs = [], [], [] for idx, l in enumerate(regions['label']): res = get_text_placement_mask(xyz, seg == l, regions['coeff'][idx], pad=2) if res is not None: mask, H, Hinv = res masks.append(mask) Hs.append(H) Hinvs.append(Hinv) filt[idx] = True regions = self.filter_regions(regions, filt) regions['place_mask'] = masks regions['homography'] = Hs regions['homography_inv'] = Hinvs return regions def warpHomography(self, src_mat, H, dst_size): dst_mat = cv2.warpPerspective(src_mat, H, dst_size, flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LINEAR) return dst_mat def homographyBB(self, bbs, H, offset=None): """ Apply homography transform to bounding-boxes. BBS: 2 x 4 x n matrix (2 coordinates, 4 points, n bbs). Returns the transformed 2x4xn bb-array. offset : a 2-tuple (dx,dy), added to points before transfomation. """ eps = 1e-16 # check the shape of the BB array: t, f, n = bbs.shape assert (t == 2) and (f == 4) # append 1 for homogenous coordinates: bbs_h = np.reshape(np.r_[bbs, np.ones((1, 4, n))], (3, 4 * n), order='F') if offset != None: bbs_h[:2, :] += np.array(offset)[:, None] # perpective: bbs_h = H.dot(bbs_h) bbs_h /= (bbs_h[2, :] + eps) bbs_h = np.reshape(bbs_h, (3, 4, n), order='F') return bbs_h[:2, :, :] def bb_filter(self, bb0, bb, text): """ Ensure that bounding-boxes are not too distorted after perspective distortion. bb0 : 2x4xn martrix of BB coordinates before perspective bb : 2x4xn matrix of BB after perspective text: string of text -- for excluding symbols/punctuations. """ h0 = np.linalg.norm(bb0[:, 3, :] - bb0[:, 0, :], axis=0) w0 = np.linalg.norm(bb0[:, 1, :] - bb0[:, 0, :], axis=0) hw0 = np.c_[h0, w0] h = np.linalg.norm(bb[:, 3, :] - bb[:, 0, :], axis=0) w = np.linalg.norm(bb[:, 1, :] - bb[:, 0, :], axis=0) hw = np.c_[h, w] # remove newlines and spaces: text = ''.join(text.split()) assert len(text) == bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) hw0 = hw0[alnum, :] hw = hw[alnum, :] min_h0, min_h = np.min(hw0[:, 0]), np.min(hw[:, 0]) asp0, asp = hw0[:, 0] / hw0[:, 1], hw[:, 0] / hw[:, 1] asp0, asp = np.median(asp0), np.median(asp) asp_ratio = asp / asp0 is_good = (min_h > self.min_char_height and asp_ratio > self.min_asp_ratio and asp_ratio < 1.0 / self.min_asp_ratio) return is_good def get_min_h(selg, bb, text): # find min-height: h = np.linalg.norm(bb[:, 3, :] - bb[:, 0, :], axis=0) # remove newlines and spaces: text = ''.join(text.split()) assert len(text) == bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) h = h[alnum] return np.min(h) def feather(self, text_mask, min_h): # determine the gaussian-blur std: if min_h <= 15: bsz = 0.25 ksz = 1 elif 15 < min_h < 30: bsz = max(0.30, 0.5 + 0.1 * np.random.randn()) ksz = 3 else: bsz = max(0.5, 1.5 + 0.5 * np.random.randn()) ksz = 5 return cv2.GaussianBlur(text_mask, (ksz, ksz), bsz) def place_text(self, rgb, collision_mask, H, Hinv): font = self.text_renderer.font_state.sample() font = self.text_renderer.font_state.init_font(font) render_res = self.text_renderer.render_sample(font, collision_mask) if render_res is None: # rendering not successful return # None else: text_mask, loc, bb, text = render_res # update the collision mask with text: collision_mask += (255 * (text_mask > 0)).astype('uint8') # warp the object mask back onto the image: # text_mask_orig = text_mask.copy() bb_orig = bb.copy() text_mask = self.warpHomography(text_mask, H, rgb.shape[:2][::-1]) bb = self.homographyBB(bb, Hinv) if not self.bb_filter(bb_orig, bb, text): # warn("bad charBB statistics") return # None # get the minimum height of the character-BB: min_h = self.get_min_h(bb, text) # feathering: text_mask = self.feather(text_mask, min_h) im_final = self.colorizer.color(rgb, [text_mask], np.array([min_h])) return im_final, text, bb, text_mask def get_num_text_regions(self, nregions): # return nregions nmax = min(self.max_text_regions, nregions) if np.random.rand() < 0.10: rnd = np.random.rand() else: rnd = np.random.beta(5.0, 1.0) return int(np.ceil(nmax * rnd)) def char2wordBB(self, charBB, text): """ Converts character bounding-boxes to word-level bounding-boxes. charBB : 2x4xn matrix of BB coordinates text : the text string output : 2x4xm matrix of BB coordinates, where, m == number of words. """ wrds = text.split() bb_idx = np.r_[0, np.cumsum([len(w) for w in wrds])] wordBB = np.zeros((2, 4, len(wrds)), 'float32') for i in range(len(wrds)): cc = charBB[:, :, bb_idx[i]:bb_idx[i + 1]] # fit a rotated-rectangle: # change shape from 2x4xn_i -> (4*n_i)x2 cc = np.squeeze(np.concatenate(np.dsplit(cc, cc.shape[-1]), axis=1)).T.astype('float32') rect = cv2.minAreaRect(cc.copy()) box = np.array(cv2.boxPoints(rect)) # find the permutation of box-coordinates which # are "aligned" appropriately with the character-bb. # (exhaustive search over all possible assignments): cc_tblr = np.c_[cc[0, :], cc[-3, :], cc[-2, :], cc[3, :]].T perm4 = np.array(list(itertools.permutations(np.arange(4)))) dists = [] for pidx in range(perm4.shape[0]): d = np.sum( np.linalg.norm(box[perm4[pidx], :] - cc_tblr, axis=1)) dists.append(d) wordBB[:, :, i] = box[perm4[np.argmin(dists)], :].T return wordBB def render_text(self, rgb, depth, seg, area, label, ninstance=1): """ This method is rendering and Args: rgb : HxWx3 image rgb values (uint8) depth : HxW depth values (float) seg : HxW segmentation region masks area : number of pixels in each region label : region labels == unique(seg) / {0} i.e., indices of pixels in SEG which constitute a region mask ninstance : number of times image should be used to place text. Returns: res : a list of dictionaries, one for each of the image instances. Each dictionary has the following structure: 'img' : rgb-image with text on it. 'bb' : 2x4xn matrix of bounding-boxes for each character in the image. 'txt' : a list of strings. 'masks': a list of masks of text placed on the image. Shape of each mask is the same as shape of original image. The correspondence b/w bb and txt is that i-th non-space white-character in txt is at bb[:,:,i]. If there's an error in pre-text placement, for e.g. if there's no suitable region for text placement, an empty list is returned. """ try: # depth -> xyz xyz = su.DepthCamera.depth2xyz(depth) # find text-regions: regions = TextRegions.get_regions(xyz, seg, area, label) # find the placement mask and homographies: regions = self.filter_for_placement(xyz, seg, regions) # finally place some text: nregions = len(regions['place_mask']) if nregions < 1: # no good region to place text on return [] except: # failure in pre-text placement # import traceback traceback.print_exc() return [] res = [] for i in range(ninstance): # place_masks - is a local copy of list of collision masks. it's updated, but is not really used. place_masks = copy.deepcopy(regions['place_mask']) print(colorize(Color.CYAN, " ** instance # : %d" % i)) idict = {'img': [], 'charBB': None, 'wordBB': None, 'txt': None} m = self.get_num_text_regions(nregions) reg_idx = np.arange(min(2 * m, nregions)) np.random.shuffle(reg_idx) reg_idx = reg_idx[:m] placed = False img = rgb.copy() itext = [] ibb = [] masks = [] # process regions: num_txt_regions = len(reg_idx) NUM_REP = 5 # re-use each region three times: reg_range = np.arange(NUM_REP * num_txt_regions) % num_txt_regions if DEBUG: print(" ... try text rendering for %s regions", len(reg_range)) for idx in reg_range: ireg = reg_idx[idx] try: if self.max_time is None: txt_render_res = self.place_text( img, place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) else: with time_limit(self.max_time): txt_render_res = self.place_text( img, place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) except TimeoutException as msg: print(msg) continue except: traceback.print_exc() # some error in placing text on the region continue if txt_render_res is not None: if DEBUG: print( " ... text rendering attempt finished successfully" ) placed = True img, text, bb, collision_mask = txt_render_res # update the region collision mask: # place_masks[ireg] = collision_mask # no point of doing that, already updated inside place_text method masks.append(collision_mask) # store the result: itext.append(text) ibb.append(bb) if placed: # at least 1 word was placed in this instance: idict['img'] = img idict['txt'] = itext idict['charBB'] = np.concatenate(ibb, axis=2) idict['wordBB'] = self.char2wordBB(idict['charBB'].copy(), ' '.join(itext)) idict['masks'] = masks idict['labeled_region'] = regions['label'] res.append(idict.copy()) return res
class RendererV3(object): def __init__(self, data_dir, max_time=None): self.text_renderer = tu.RenderFont(data_dir) self.colorizer = Colorize(data_dir) #self.colorizerV2 = colorV2.Colorize(data_dir) self.min_char_height = 8 #px self.min_asp_ratio = 0.4 # self.max_text_regions = 7 self.max_time = max_time def filter_regions(self,regions,filt): """ filt : boolean list of regions to keep. """ idx = np.arange(len(filt))[filt] for k in list(regions.keys()): regions[k] = [regions[k][i] for i in idx] return regions def filter_for_placement(self,xyz,seg,regions): filt = np.zeros(len(regions['label'])).astype('bool') masks,Hs,Hinvs = [],[], [] for idx,l in enumerate(regions['label']): res = get_text_placement_mask(xyz,seg==l,regions['coeff'][idx],pad=2) if res is not None: mask,H,Hinv = res masks.append(mask) Hs.append(H) Hinvs.append(Hinv) filt[idx] = True regions = self.filter_regions(regions,filt) regions['place_mask'] = masks regions['homography'] = Hs regions['homography_inv'] = Hinvs return regions def warpHomography(self,src_mat,H,dst_size): dst_mat = cv2.warpPerspective(src_mat, H, dst_size, flags=cv2.WARP_INVERSE_MAP|cv2.INTER_LINEAR) return dst_mat def homographyBB(self, bbs, H, offset=None): """ Apply homography transform to bounding-boxes. BBS: 2 x 4 x n matrix (2 coordinates, 4 points, n bbs). Returns the transformed 2x4xn bb-array. offset : a 2-tuple (dx,dy), added to points before transfomation. """ eps = 1e-16 # check the shape of the BB array: t,f,n = bbs.shape assert (t==2) and (f==4) # append 1 for homogenous coordinates: bbs_h = np.reshape(np.r_[bbs, np.ones((1,4,n))], (3,4*n), order='F') if offset != None: bbs_h[:2,:] += np.array(offset)[:,None] # perpective: bbs_h = H.dot(bbs_h) bbs_h /= (bbs_h[2,:]+eps) bbs_h = np.reshape(bbs_h, (3,4,n), order='F') return bbs_h[:2,:,:] def bb_filter(self,bb0,bb,text): """ Ensure that bounding-boxes are not too distorted after perspective distortion. bb0 : 2x4xn martrix of BB coordinates before perspective bb : 2x4xn matrix of BB after perspective text: string of text -- for excluding symbols/punctuations. """ h0 = np.linalg.norm(bb0[:,3,:] - bb0[:,0,:], axis=0) w0 = np.linalg.norm(bb0[:,1,:] - bb0[:,0,:], axis=0) hw0 = np.c_[h0,w0] h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) w = np.linalg.norm(bb[:,1,:] - bb[:,0,:], axis=0) hw = np.c_[h,w] # remove newlines and spaces: text = ''.join(text.split()) assert len(text)==bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) hw0 = hw0[alnum,:] hw = hw[alnum,:] min_h0, min_h = np.min(hw0[:,0]), np.min(hw[:,0]) asp0, asp = hw0[:,0]/hw0[:,1], hw[:,0]/hw[:,1] asp0, asp = np.median(asp0), np.median(asp) asp_ratio = asp/asp0 is_good = ( min_h > self.min_char_height and asp_ratio > self.min_asp_ratio and asp_ratio < 1.0/self.min_asp_ratio) return is_good def get_min_h(selg, bb, text): # find min-height: h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) # remove newlines and spaces: text = ''.join(text.split()) assert len(text)==bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) h = h[alnum] return np.min(h) def feather(self, text_mask, min_h): # determine the gaussian-blur std: if min_h <= 15 : bsz = 0.25 ksz=1 elif 15 < min_h < 30: bsz = max(0.30, 0.5 + 0.1*np.random.randn()) ksz = 3 else: bsz = max(0.5, 1.5 + 0.5*np.random.randn()) ksz = 5 return cv2.GaussianBlur(text_mask,(ksz,ksz),bsz) def place_text(self,rgb,collision_mask,H,Hinv): font = self.text_renderer.font_state.sample() font = self.text_renderer.font_state.init_font(font) render_res = self.text_renderer.render_sample(font,collision_mask) if render_res is None: # rendering not successful return #None else: text_mask,loc,bb,text = render_res #text=text.decode('utf-8') # update the collision mask with text: collision_mask += (255 * (text_mask>0)).astype('uint8') # warp the object mask back onto the image: text_mask_orig = text_mask.copy() bb_orig = bb.copy() text_mask = self.warpHomography(text_mask,H,rgb.shape[:2][::-1]) bb = self.homographyBB(bb,Hinv) if not self.bb_filter(bb_orig,bb,text): colorize(Color.RED, 'bad charBB statistics') #warn("bad charBB statistics") return #None # get the minimum height of the character-BB: min_h = self.get_min_h(bb,text) #feathering: text_mask = self.feather(text_mask, min_h) im_final = self.colorizer.color(rgb,[text_mask],np.array([min_h])) print(colorize(Color.GREEN, 'text in synthgen.py/place_text to return '+text)) return im_final, text, bb, collision_mask def get_num_text_regions(self, nregions): #return nregions nmax = min(self.max_text_regions, nregions) if np.random.rand() < 0.10: rnd = np.random.rand() else: rnd = np.random.beta(5.0,1.0) return int(np.ceil(nmax * rnd)) def char2wordBB(self, charBB, text): """ Converts character bounding-boxes to word-level bounding-boxes. charBB : 2x4xn matrix of BB coordinates text : the text string output : 2x4xm matrix of BB coordinates, where, m == number of words. """ wrds = text.split() bb_idx = np.r_[0, np.cumsum([len(w) for w in wrds])] wordBB = np.zeros((2,4,len(wrds)), 'float32') for i in range(len(wrds)): cc = charBB[:,:,bb_idx[i]:bb_idx[i+1]] # fit a rotated-rectangle: # change shape from 2x4xn_i -> (4*n_i)x2 cc = np.squeeze(np.concatenate(np.dsplit(cc,cc.shape[-1]),axis=1)).T.astype('float32') rect = cv2.minAreaRect(cc.copy()) box = np.array(cv2.boxPoints(rect)) # find the permutation of box-coordinates which # are "aligned" appropriately with the character-bb. # (exhaustive search over all possible assignments): cc_tblr = np.c_[cc[0,:], cc[-3,:], cc[-2,:], cc[3,:]].T perm4 = np.array(list(itertools.permutations(np.arange(4)))) dists = [] for pidx in range(perm4.shape[0]): d = np.sum(np.linalg.norm(box[perm4[pidx],:]-cc_tblr,axis=1)) dists.append(d) wordBB[:,:,i] = box[perm4[np.argmin(dists)],:].T return wordBB def render_text(self,rgb,depth,seg,area,label,ninstance=1,viz=False): """ rgb : HxWx3 image rgb values (uint8) depth : HxW depth values (float) seg : HxW segmentation region masks area : number of pixels in each region label : region labels == unique(seg) / {0} i.e., indices of pixels in SEG which constitute a region mask ninstance : no of times image should be used to place text. @return: res : a list of dictionaries, one for each of the image instances. Each dictionary has the following structure: 'img' : rgb-image with text on it. 'bb' : 2x4xn matrix of bounding-boxes for each character in the image. 'txt' : a list of strings. The correspondence b/w bb and txt is that i-th non-space white-character in txt is at bb[:,:,i]. If there's an error in pre-text placement, for e.g. if there's no suitable region for text placement, an empty list is returned. """ try: # depth -> xyz xyz = su.DepthCamera.depth2xyz(depth) # find text-regions: regions = TextRegions.get_regions(xyz,seg,area,label) # find the placement mask and homographies: regions = self.filter_for_placement(xyz,seg,regions) # finally place some text: nregions = len(regions['place_mask']) if nregions < 1: # no good region to place text on return [] except: # failure in pre-text placement #import traceback traceback.print_exc() return [] res = [] for i in range(ninstance): place_masks = copy.deepcopy(regions['place_mask']) print(colorize(Color.CYAN, " ** instance # : %d"%i)) idict = {'img':[], 'charBB':None, 'wordBB':None, 'txt':None} m = self.get_num_text_regions(nregions)#np.arange(nregions)#min(nregions, 5*ninstance*self.max_text_regions)) reg_idx = np.arange(min(2*m,nregions)) #np.random.shuffle(reg_idx) reg_idx = reg_idx[:m] placed = False img = rgb.copy() itext = [] ibb = [] # process regions: num_txt_regions = len(reg_idx) NUM_REP = 5 # re-use each region three times: reg_range = np.arange(NUM_REP * num_txt_regions) % num_txt_regions for idx in reg_range: ireg = reg_idx[idx] try: if self.max_time is None: txt_render_res = self.place_text(img,place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) else: with time_limit(self.max_time): txt_render_res = self.place_text(img,place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) except TimeoutException as msg: print(msg) continue except: traceback.print_exc() # some error in placing text on the region continue if txt_render_res is not None: placed = True img,text,bb,collision_mask = txt_render_res # update the region collision mask: place_masks[ireg] = collision_mask # store the result: itext.append(text) ibb.append(bb) print(colorize(Color.GREEN, 'text in synthgen.py/render_text append into itext '+text)) if placed: # at least 1 word was placed in this instance: idict['img'] = img idict['txt'] = itext idict['charBB'] = np.concatenate(ibb, axis=2) idict['wordBB'] = self.char2wordBB(idict['charBB'].copy(), ' '.join(itext)) print(colorize(Color.GREEN, itext)) res.append(idict.copy()) if viz: viz_textbb(1,img, [idict['wordBB']], alpha=1.0) viz_masks(2,img,seg,depth,regions['label']) # viz_regions(rgb.copy(),xyz,seg,regions['coeff'],regions['label']) if i < ninstance-1: input(colorize(Color.BLUE,'continue?',True)) return res
class RendererV3(object): def __init__(self, data_dir, max_time=None, logos=None): self.text_renderer = tu.RenderFont(data_dir) self.colorizer = Colorize(data_dir) #self.colorizerV2 = colorV2.Colorize(data_dir) self.logos = logos self.min_char_height = 8 #px self.min_asp_ratio = 0.4 # self.max_text_regions = 7 self.max_time = max_time def filter_regions(self,regions,filt): """ filt : boolean list of regions to keep. """ idx = np.arange(len(filt))[filt] for k in regions.keys(): regions[k] = [regions[k][i] for i in idx] return regions def filter_for_placement(self,xyz,seg,regions): filt = np.zeros(len(regions['label'])).astype('bool') masks,Hs,Hinvs = [],[], [] for idx,l in enumerate(regions['label']): res = get_text_placement_mask(xyz,seg==l,regions['coeff'][idx],pad=2) if res is not None: mask,H,Hinv = res masks.append(mask) Hs.append(H) Hinvs.append(Hinv) filt[idx] = True regions = self.filter_regions(regions,filt) regions['place_mask'] = masks regions['homography'] = Hs regions['homography_inv'] = Hinvs return regions def warpHomography(self,src_mat,H,dst_size): dst_mat = cv2.warpPerspective(src_mat, H, dst_size, flags=cv2.WARP_INVERSE_MAP|cv2.INTER_LINEAR) return dst_mat def homographyBB(self, bbs, H, offset=None): """ Apply homography transform to bounding-boxes. BBS: 2 x 4 x n matrix (2 coordinates, 4 points, n bbs). Returns the transformed 2x4xn bb-array. offset : a 2-tuple (dx,dy), added to points before transfomation. """ eps = 1e-16 # check the shape of the BB array: t,f,n = bbs.shape assert (t==2) and (f==4) # append 1 for homogenous coordinates: bbs_h = np.reshape(np.r_[bbs, np.ones((1,4,n))], (3,4*n), order='F') if offset != None: bbs_h[:2,:] += np.array(offset)[:,None] # perpective: bbs_h = H.dot(bbs_h) bbs_h /= (bbs_h[2,:]+eps) bbs_h = np.reshape(bbs_h, (3,4,n), order='F') return bbs_h[:2,:,:] def bb_filter(self,bb0,bb,text): """ Ensure that bounding-boxes are not too distorted after perspective distortion. bb0 : 2x4xn martrix of BB coordinates before perspective bb : 2x4xn matrix of BB after perspective text: string of text -- for excluding symbols/punctuations. """ h0 = np.linalg.norm(bb0[:,3,:] - bb0[:,0,:], axis=0) w0 = np.linalg.norm(bb0[:,1,:] - bb0[:,0,:], axis=0) hw0 = np.c_[h0,w0] h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) w = np.linalg.norm(bb[:,1,:] - bb[:,0,:], axis=0) hw = np.c_[h,w] # remove newlines and spaces: text = ''.join(text.split()) assert len(text)==bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) hw0 = hw0[alnum,:] hw = hw[alnum,:] min_h0, min_h = np.min(hw0[:,0]), np.min(hw[:,0]) asp0, asp = hw0[:,0]/hw0[:,1], hw[:,0]/hw[:,1] asp0, asp = np.median(asp0), np.median(asp) asp_ratio = asp/asp0 is_good = ( min_h > self.min_char_height and asp_ratio > self.min_asp_ratio and asp_ratio < 1.0/self.min_asp_ratio) return is_good def get_min_h(selg, bb, text): # find min-height: h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) # remove newlines and spaces: text = ''.join(text.split()) assert len(text)==bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) h = h[alnum] return np.min(h) def feather(self, text_mask, min_h): # determine the gaussian-blur std: if min_h <= 15 : bsz = 0.25 ksz=1 elif 15 < min_h < 30: bsz = max(0.30, 0.5 + 0.1*np.random.randn()) ksz = 3 else: bsz = max(0.5, 1.5 + 0.5*np.random.randn()) ksz = 5 return cv2.GaussianBlur(text_mask,(ksz,ksz),bsz) def robust_HW(self,mask): m = mask.copy() m = (~mask).astype('float')/255 rH = np.median(np.sum(m,axis=0)) rW = np.median(np.sum(m,axis=1)) return rH,rW def sample_font_height_px(self,h_min,h_max): if np.random.rand() < 0.10: rnd = np.random.rand() else: rnd = np.random.beta(2.0,2.0) h_range = h_max - h_min f_h = np.floor(h_min + h_range*rnd) return f_h def bb_xywh2coords(self,bbs): """ Takes an nx4 bounding-box matrix specified in x,y,w,h format and outputs a 2x4xn bb-matrix, (4 vertices per bb). """ n,_ = bbs.shape coords = np.zeros((2,4,n)) for i in range(n): coords[:,:,i] = bbs[i,:2][:,None] coords[0,1,i] += bbs[i,2] coords[:,2,i] += bbs[i,2:4] coords[1,3,i] += bbs[i,3] return coords def actually_place_logo(self, text_arrs, back_arr, bbs, logos): areas = [-np.prod(ta.shape) for ta in text_arrs] order = np.argsort(areas) locs = [None for i in range(len(text_arrs))] out_arr = np.zeros_like(back_arr) logo_out = np.zeros((back_arr.shape[0], back_arr.shape[1], logos[0].shape[2]), dtype=np.float32) for i in order: ba = np.clip(back_arr.copy().astype(np.float), 0, 255) ta = np.clip(text_arrs[i].copy().astype(np.float), 0, 255) ba[ba > 127] = 1e8 intersect = ssig.fftconvolve(ba,ta[::-1,::-1],mode='valid') safemask = intersect < 1e8 if not np.any(safemask): # no collision-free position: #warn("COLLISION!!!") return back_arr,locs[:i],bbs[:i],order[:i], None minloc = np.transpose(np.nonzero(safemask)) loc = minloc[np.random.choice(minloc.shape[0]),:] locs[i] = loc # update the bounding-boxes: bbs[i] = bbs[i] + loc[::-1][:,None,None] # blit the text onto the canvas w,h = text_arrs[i].shape out_arr[loc[0]:loc[0]+w,loc[1]:loc[1]+h] += text_arrs[i] logo_out[loc[0]:loc[0]+w,loc[1]:loc[1]+h] += logos[i] return out_arr, locs, bbs, order, logo_out def pick_random_logos(self): im = None while im is None: name = self.logos[np.random.randint(low=0, high=len(self.logos))] im = cv2.imread(name, cv2.IMREAD_UNCHANGED) short_name = osp.basename(name) return im, short_name def actually_render_logo(self, mask): """ Places text in the "collision-free" region as indicated in the mask -- 255 for unsafe, 0 for safe. The text is rendered using FONT, the text content is TEXT. """ #H,W = mask.shape H,W = self.robust_HW(mask) # f_asp = self.font_state.get_aspect_ratio(font) min_logo_h = 40 max_logo_h = 300 max_shrink_trials = 50 # let's just place one text-instance for now ## TODO : change this to allow multiple text instances? i = 0 print ("Attempt") while i < max_shrink_trials and max_logo_h > min_logo_h: # if i > 0: # print colorize(Color.BLUE, "shrinkage trial : %d"%i, True) # sample a random font-height: logo_h_px = self.sample_font_height_px(min_logo_h, max_logo_h) #print "font-height : %.2f (min: %.2f, max: %.2f)"%(f_h_px, self.min_font_h,max_font_h) # update for the loop max_logo_h = logo_h_px i += 1 # compute the max-number of lines/chars-per-line: # nline, nchar = self.get_nline_nchar(mask.shape[:2],f_h,f_h*f_asp) # #print " > nline = %d, nchar = %d"%(nline, nchar) # # assert nline >= 1 and nchar >= self.min_nchar # sample text: # text_type = sample_weighted(self.p_text) # text = self.text_source.sample(nline,nchar,text_type) # if len(text)==0 or np.any([len(line)==0 for line in text]): # continue #print colorize(Color.GREEN, text) logo, name = self.pick_random_logos() # calculate logo width logo_w = logo_h_px * (float(logo.shape[1]) / logo.shape[0]) print(logo_w, logo_h_px) if logo_w >= W: continue # render the logo if len(logo.shape) == 3 and logo.shape[2] == 4: logo_arr = logo[:, :, 3] else: logo_arr = np.ones((logo.shape[0], logo.shape[1])) logo_arr = cv2.resize(logo_arr, (int(logo_w), int(logo_h_px))) logo = cv2.resize(logo, (int(logo_w), int(logo_h_px))) logo_bb = np.array([[0, 0, logo_arr.shape[1], logo_arr.shape[0]]]) bb = self.bb_xywh2coords(logo_bb) # make sure that the text-array is not bigger than mask array: if np.any(np.r_[logo_arr.shape[:2]] > np.r_[mask.shape[:2]]): # warn("text-array is bigger than mask") continue # position the text within the mask: # logger.debug(VisualRecord("original logo", [logo, logo_arr])) logo_mask,loc, bb, _, logo = self.actually_place_logo([logo_arr], mask, [bb], [logo]) # logger.debug(VisualRecord("positioned logo", [logo, logo_mask])) if len(loc) > 0: # successful in placing the text collision-free: return logo_mask,loc[0],bb[0], name, logo return #None def place_logo(self, rgb, collision_mask, H, Hinv): # logo = cv2.imread("logos/nike.png", cv2.IMREAD_UNCHANGED) # name = "nike" render_res = self.actually_render_logo(collision_mask) # self.text_renderer.render_sample(font, collision_mask) if render_res is None: # rendering not successful return # None else: logo_mask, loc, bb, logo_name, logo_detail = render_res # update the collision mask with text: collision_mask += (255 * (logo_mask > 0)).astype('uint8') # warp the object mask back onto the image: logo_mask_orig = logo_mask.copy() bb_orig = bb.copy() # logger.debug(VisualRecord("logo_pre_warp", [logo_mask, logo_detail])) logo_mask = self.warpHomography(logo_mask, H, rgb.shape[:2][::-1]) logo_detail = self.warpHomography(logo_detail, H, rgb.shape[:2][::-1]) # print "logo mask detail", logo_mask.shape, logo_detail.shape bb = self.homographyBB(bb, Hinv) # if not self.bb_filter(bb_orig, bb, logo_name): # # warn("bad charBB statistics") # return # None # get the minimum height of the character-BB: # min_h = self.get_min_h(bb, [logo_name]) min_h = np.abs(bb[0, 3, :] - bb[0, 0, :]) # feathering: logo_mask = self.feather(logo_mask, min_h) # print "logo mask detail2", logo_mask.shape, logo_detail.shape # logger.debug(VisualRecord("logo", [logo_mask, logo_detail])) #TODO deal with grayscale/color logos here im_final = self.colorizer.color(rgb, [logo_mask], np.array([min_h]), logo_detail=[logo_detail]) # bbox_image = im_final.copy() # for bbox in bb.copy().T: # print "bbox", bbox # for coord in bbox: # cv2.circle(bbox_image, (int(coord[0]), int(coord[1])), 5, (255,0,0), thickness=-1) # cv2.circle(bbox_image, (bbox[0], bbox[2]), 5, (255, 0, 0), thickness=-1) # cv2.circle(bbox_image, (bbox[3], bbox[2]), 5, (255, 0, 0), thickness=-1) # cv2.circle(bbox_image, (bbox[3], bbox[1]), 5, (255, 0, 0), thickness=-1) # logger.debug(VisualRecord("bboxes", imgs=[bbox_image])) # im_final = rgb.copy() return im_final, logo_name, bb, collision_mask def place_text(self,rgb,collision_mask,H,Hinv): font = self.text_renderer.font_state.sample() font = self.text_renderer.font_state.init_font(font) render_res = self.text_renderer.render_sample(font,collision_mask) if render_res is None: # rendering not successful return #None else: text_mask,loc,bb,text = render_res # update the collision mask with text: collision_mask += (255 * (text_mask>0)).astype('uint8') # warp the object mask back onto the image: text_mask_orig = text_mask.copy() bb_orig = bb.copy() text_mask = self.warpHomography(text_mask,H,rgb.shape[:2][::-1]) bb = self.homographyBB(bb,Hinv) if not self.bb_filter(bb_orig,bb,text): #warn("bad charBB statistics") return #None # get the minimum height of the character-BB: min_h = self.get_min_h(bb,text) #feathering: text_mask = self.feather(text_mask, min_h) im_final = self.colorizer.color(rgb,[text_mask],np.array([min_h])) return im_final, text, bb, collision_mask def get_num_text_regions(self, nregions): #return nregions nmax = min(self.max_text_regions, nregions) if np.random.rand() < 0.10: rnd = np.random.rand() else: rnd = np.random.beta(5.0,1.0) return int(np.ceil(nmax * rnd)) def char2wordBB(self, charBB, text): """ Converts character bounding-boxes to word-level bounding-boxes. charBB : 2x4xn matrix of BB coordinates text : the text string output : 2x4xm matrix of BB coordinates, where, m == number of words. """ wrds = text.split() bb_idx = np.r_[0, np.cumsum([len(w) for w in wrds])] wordBB = np.zeros((2,4,len(wrds)), 'float32') for i in range(len(wrds)): cc = charBB[:,:,bb_idx[i]:bb_idx[i+1]] # fit a rotated-rectangle: # change shape from 2x4xn_i -> (4*n_i)x2 cc = np.squeeze(np.concatenate(np.dsplit(cc,cc.shape[-1]),axis=1)).T.astype('float32') rect = cv2.minAreaRect(cc.copy()) box = np.array(cv2.boxPoints(rect)) # find the permutation of box-coordinates which # are "aligned" appropriately with the character-bb. # (exhaustive search over all possible assignments): cc_tblr = np.c_[cc[0,:], cc[-3,:], cc[-2,:], cc[3,:]].T perm4 = np.array(list(itertools.permutations(np.arange(4)))) dists = [] for pidx in range(perm4.shape[0]): d = np.sum(np.linalg.norm(box[perm4[pidx],:]-cc_tblr,axis=1)) dists.append(d) wordBB[:,:,i] = box[perm4[np.argmin(dists)],:].T return wordBB def render_text(self,rgb,depth,seg,area,label,ninstance=1,viz=False): """ rgb : HxWx3 image rgb values (uint8) depth : HxW depth values (float) seg : HxW segmentation region masks area : number of pixels in each region label : region labels == unique(seg) / {0} i.e., indices of pixels in SEG which constitute a region mask ninstance : no of times image should be used to place text. @return: res : a list of dictionaries, one for each of the image instances. Each dictionary has the following structure: 'img' : rgb-image with text on it. 'bb' : 2x4xn matrix of bounding-boxes for each character in the image. 'txt' : a list of strings. The correspondence b/w bb and txt is that i-th non-space white-character in txt is at bb[:,:,i]. If there's an error in pre-text placement, for e.g. if there's no suitable region for text placement, an empty list is returned. """ try: # depth -> xyz xyz = su.DepthCamera.depth2xyz(depth) # find text-regions: regions = TextRegions.get_regions(xyz,seg,area,label) # find the placement mask and homographies: regions = self.filter_for_placement(xyz,seg,regions) # finally place some text: nregions = len(regions['place_mask']) if nregions < 1: # no good region to place text on return [] except: # failure in pre-text placement #import traceback traceback.print_exc() return [] res = [] for i in range(ninstance): place_masks = copy.deepcopy(regions['place_mask']) print (colorize(Color.CYAN, " ** instance # : %d"%i)) idict = {'img':[], 'charBB':None, 'wordBB':None, 'txt':None} m = self.get_num_text_regions(nregions)#np.arange(nregions)#min(nregions, 5*ninstance*self.max_text_regions)) reg_idx = np.arange(min(2*m,nregions)) np.random.shuffle(reg_idx) reg_idx = reg_idx[:m] placed = False img = rgb.copy() itext = [] ibb = [] # process regions: num_txt_regions = len(reg_idx) NUM_REP = 5 # re-use each region three times: reg_range = np.arange(NUM_REP * num_txt_regions) % num_txt_regions for idx in reg_range: ireg = reg_idx[idx] try: if self.max_time is None: txt_render_res = self.place_text(img,place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) else: with time_limit(self.max_time): txt_render_res = self.place_text(img,place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) except TimeoutException as msg: print (msg) continue except: traceback.print_exc() # some error in placing text on the region continue if txt_render_res is not None: placed = True img,text,bb,collision_mask = txt_render_res # update the region collision mask: place_masks[ireg] = collision_mask # store the result: itext.append(text) ibb.append(bb) if placed: # at least 1 word was placed in this instance: idict['img'] = img idict['txt'] = itext idict['charBB'] = np.concatenate(ibb, axis=2) idict['wordBB'] = self.char2wordBB(idict['charBB'].copy(), ' '.join(itext)) res.append(idict.copy()) if viz: viz_textbb(1,img, [idict['wordBB']], alpha=1.0) viz_masks(2,img,seg,depth,regions['label']) # viz_regions(rgb.copy(),xyz,seg,regions['coeff'],regions['label']) if i < ninstance-1: raw_input(colorize(Color.BLUE,'continue?',True)) return res def render_logo(self,rgb,depth, seg, area, label, ninstance=1, viz=False): print("hold") """ rgb : HxWx3 image rgb values (uint8) depth : HxW depth values (float) seg : HxW segmentation region masks area : number of pixels in each region label : region labels == unique(seg) / {0} i.e., indices of pixels in SEG which constitute a region mask ninstance : no of times image should be used to place text. @return: res : a list of dictionaries, one for each of the image instances. Each dictionary has the following structure: 'img' : rgb-image with text on it. 'bb' : 2x4xn matrix of bounding-boxes for each character in the image. 'txt' : a list of strings. The correspondence b/w bb and txt is that i-th non-space white-character in txt is at bb[:,:,i]. If there's an error in pre-text placement, for e.g. if there's no suitable region for text placement, an empty list is returned. """ try: # depth -> xyz xyz = su.DepthCamera.depth2xyz(depth) # find text-regions: regions = TextRegions.get_regions(xyz, seg, area, label) # find the placement mask and homographies: regions = self.filter_for_placement(xyz, seg, regions) # finally place some text: nregions = len(regions['place_mask']) if nregions < 1: # no good region to place text on return [] except: # failure in pre-text placement # import traceback traceback.print_exc() return [] res = [] for i in range(ninstance): place_masks = copy.deepcopy(regions['place_mask']) print colorize(Color.CYAN, " ** instance # : %d" % i) idict = {'img': [], 'logoBB': None, 'logoName': None} m = self.get_num_text_regions(nregions) reg_idx = np.arange(min(2 * m, nregions)) np.random.shuffle(reg_idx) reg_idx = reg_idx[:m] placed = False img = rgb.copy() ilogo = [] ibb = [] # process regions: num_txt_regions = len(reg_idx) NUM_REP = np.random.randint(low=1, high=4) # re-use each region three times: reg_range = np.arange(NUM_REP * num_txt_regions) % num_txt_regions for idx in reg_range: ireg = reg_idx[idx] try: if self.max_time is None: txt_render_res = self.place_logo(img, place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) else: with time_limit(self.max_time): txt_render_res = self.place_logo(img, place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) except TimeoutException as msg: print(msg) continue except: traceback.print_exc() #some error in placing text on the region continue if txt_render_res is not None: placed = True img, logo, bb, collision_mask = txt_render_res # update the region collision mask: place_masks[ireg] = collision_mask # store the result: ilogo.append(logo) ibb.append(bb) if placed: # at least 1 word was placed in this instance: idict['img'] = img idict['logoName'] = ilogo idict['logoBB'] = np.concatenate(ibb, axis=2) res.append(idict.copy()) # if viz: # viz_textbb(1, img, [idict['logobb']], alpha=1.0) # # viz_masks(2, img, seg, depth, regions['label']) # # viz_regions(rgb.copy(),xyz,seg,regions['coeff'],regions['label']) # if i < ninstance - 1: # raw_input(colorize(Color.BLUE, 'continue?', True)) return res
class RendererV3(object): def __init__(self, data_dir, max_time=None, text_number=1, font_color_user = None, font_size_user = 18, text_content_user = "******") : self.text_renderer = tu.RenderFont( data_dir, font_size_user, text_content_user) self.colorizer = Colorize(data_dir, font_color= font_color_user) self.NUM_REP = text_number self.min_char_height = 8 #px self.min_asp_ratio = 0.4 # self.max_text_regions = 7 self.max_time = max_time def filter_regions(self,regions,filt): """ filt : boolean list of regions to keep. """ idx = np.arange(len(filt))[filt] for k in regions.keys(): regions[k] = [regions[k][i] for i in idx] return regions def filter_for_placement(self,xyz,seg,regions): filt = np.zeros(len(regions['label'])).astype('bool') masks,Hs,Hinvs = [],[], [] for idx,l in enumerate(regions['label']): res = get_text_placement_mask(xyz,seg==l,regions['coeff'][idx],pad=2) if res is not None: mask,H,Hinv = res masks.append(mask) Hs.append(H) Hinvs.append(Hinv) filt[idx] = True regions = self.filter_regions(regions,filt) regions['place_mask'] = masks regions['homography'] = Hs regions['homography_inv'] = Hinvs return regions def warpHomography(self,src_mat,H,dst_size): dst_mat = cv2.warpPerspective(src_mat, H, dst_size, flags=cv2.WARP_INVERSE_MAP|cv2.INTER_LINEAR) return dst_mat def homographyBB(self, bbs, H, offset=None): """ Apply homography transform to bounding-boxes. BBS: 2 x 4 x n matrix (2 coordinates, 4 points, n bbs). Returns the transformed 2x4xn bb-array. offset : a 2-tuple (dx,dy), added to points before transfomation. """ eps = 1e-16 # check the shape of the BB array: t,f,n = bbs.shape assert (t==2) and (f==4) # append 1 for homogenous coordinates: bbs_h = np.reshape(np.r_[bbs, np.ones((1,4,n))], (3,4*n), order='F') if offset != None: bbs_h[:2,:] += np.array(offset)[:,None] # perpective: bbs_h = H.dot(bbs_h) bbs_h /= (bbs_h[2,:]+eps) bbs_h = np.reshape(bbs_h, (3,4,n), order='F') return bbs_h[:2,:,:] def bb_filter(self,bb0,bb,text): """ Ensure that bounding-boxes are not too distorted after perspective distortion. bb0 : 2x4xn martrix of BB coordinates before perspective bb : 2x4xn matrix of BB after perspective text: string of text -- for excluding symbols/punctuations. """ h0 = np.linalg.norm(bb0[:,3,:] - bb0[:,0,:], axis=0) w0 = np.linalg.norm(bb0[:,1,:] - bb0[:,0,:], axis=0) hw0 = np.c_[h0,w0] h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) w = np.linalg.norm(bb[:,1,:] - bb[:,0,:], axis=0) hw = np.c_[h,w] # remove newlines and spaces: text = ''.join(text.split()) assert len(text)==bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) hw0 = hw0[alnum,:] hw = hw[alnum,:] min_h0, min_h = np.min(hw0[:,0]), np.min(hw[:,0]) asp0, asp = hw0[:,0]/hw0[:,1], hw[:,0]/hw[:,1] asp0, asp = np.median(asp0), np.median(asp) asp_ratio = asp/asp0 is_good = ( min_h > self.min_char_height and asp_ratio > self.min_asp_ratio and asp_ratio < 1.0/self.min_asp_ratio) return is_good def get_min_h(selg, bb, text): # find min-height: h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) # remove newlines and spaces: text = ''.join(text.split()) assert len(text)==bb.shape[-1] alnum = np.array([ch.isalnum() for ch in text]) h = h[alnum] return np.min(h) def feather(self, text_mask, min_h): # determine the gaussian-blur std: if min_h <= 15 : bsz = 0.25 ksz=1 elif 15 < min_h < 30: bsz = max(0.30, 0.5 + 0.1*np.random.randn()) ksz = 3 else: bsz = max(0.5, 1.5 + 0.5*np.random.randn()) ksz = 5 return cv2.GaussianBlur(text_mask,(ksz,ksz),bsz) def place_text(self, rgb, collision_mask): font = self.text_renderer.font_state.sample() font = self.text_renderer.font_state.init_font(font) render_res = self.text_renderer.render_sample(font,collision_mask) if render_res is None: # rendering not successful return #None else: text_mask,loc,bb,text = render_res # update the collision mask with text: kernel = np.ones((32, 32), np.uint8) # 2*self.min_font_h mask_ext = cv2.dilate(text_mask,kernel,iterations = 1) collision_mask += (255 * (mask_ext > 0)).astype('uint8') # warp the object mask back onto the image: # text_mask_orig = text_mask.copy() # bb_orig = bb.copy() # text_mask = self.warpHomography(text_mask,H,rgb.shape[:2][::-1]) # bb = self.homographyBB(bb,Hinv) # # if not self.bb_filter(bb_orig,bb,text): # #warn("bad charBB statistics") # return #None # get the minimum height of the character-BB: min_h = self.get_min_h(bb,text) #feathering: # plt.imshow(text_mask) # text_mask = self.feather(text_mask, min_h) # plt.imshow(text_mask) im_final = self.colorizer.color(rgb,[text_mask],np.array([min_h])) return im_final, text, bb, collision_mask def get_num_text_regions(self, nregions): #return nregions nmax = min(self.max_text_regions, nregions) if np.random.rand() < 0.10: rnd = np.random.rand() else: rnd = np.random.beta(5.0,1.0) return int(np.ceil(nmax * rnd)) def char2wordBB(self, charBB, text): """ Converts character bounding-boxes to word-level bounding-boxes. charBB : 2x4xn matrix of BB coordinates text : the text string output : 2x4xm matrix of BB coordinates, where, m == number of words. """ wrds = text.split() bb_idx = np.r_[0, np.cumsum([len(w) for w in wrds])] wordBB = np.zeros((2,4,len(wrds)), 'float32') for i in xrange(len(wrds)): cc = charBB[:,:,bb_idx[i]:bb_idx[i+1]] # fit a rotated-rectangle: # change shape from 2x4xn_i -> (4*n_i)x2 cc = np.squeeze(np.concatenate(np.dsplit(cc,cc.shape[-1]),axis=1)).T.astype('float32') rect = cv2.minAreaRect(cc.copy()) box = np.array(cv2.cv.BoxPoints(rect)) # find the permutation of box-coordinates which # are "aligned" appropriately with the character-bb. # (exhaustive search over all possible assignments): cc_tblr = np.c_[cc[0,:], cc[-3,:], cc[-2,:], cc[3,:]].T perm4 = np.array(list(itertools.permutations(np.arange(4)))) dists = [] for pidx in xrange(perm4.shape[0]): d = np.sum(np.linalg.norm(box[perm4[pidx],:]-cc_tblr,axis=1)) dists.append(d) wordBB[:,:,i] = box[perm4[np.argmin(dists)],:].T return wordBB def char2lineBB(self, charBB, itext, imgsize): """ Converts character bounding-boxes to line-level bounding-boxes. charBB : 2x4xn matrix of BB coordinates text : the text string output : 2x4xm matrix of BB coordinates, where, m == number of lines. """ lt = len(itext) text = ' '.join(itext) lines = text.split('\n') lineBB = np.zeros((2, 4, len(lines)+lt-1), 'float32') wrds = text.split() bb_idx = np.r_[0, np.cumsum([len(w) for w in wrds])] nlines = 0 idx1 = 0 idx2 = 0 txt = [] letterBB = [] for it in range(lt): text = itext[it] lines = text.split('\n') for i in range(len(lines)): wrds = lines[i].split() lw = len(wrds) if lw <= 0 : continue idx2 += lw cc = charBB[:, :, bb_idx[idx1]:bb_idx[idx2]] idx1 = idx2 # cc = np.squeeze(np.concatenate(np.dsplit(cc, cc.shape[-1]), axis=1)).T.astype('float32') coor_min = np.min(cc, axis=0) coor_max = np.max(cc, axis=0) ext = np.clip(np.round(np.random.rand(2) * 10), None, 6) #not exceed self.min_font_h coor_min[0] = np.clip(coor_min[0] - ext[0], 0, None) coor_min[1] = np.clip(coor_min[1] - ext[1], 0, None) coor_max[0] = np.clip(coor_max[0] + ext[0], None, imgsize[1] - 1) coor_max[1] = np.clip(coor_max[1] + ext[1], None, imgsize[0] - 1) box = np.array([[coor_min[0], coor_min[1]], [coor_min[0], coor_max[1]], [coor_max[0], coor_max[1]], [coor_max[0], coor_min[1]]]) lineBB[:, :, nlines] = box.T nlines += 1 linetxt = lines[i].strip() linetxt = linetxt.strip('\t') txt.append(linetxt) letterBB.append(cc.astype(int)) return lineBB[:, :, 0:nlines], txt, letterBB def render_text(self, rgb, place_mask, dir, ninstance=1, viz=False): """ rgb : HxWx3 image rgb values (uint8) depth : HxW depth values (float) seg : HxW segmentation region masks area : number of pixels in each region label : region labels == unique(seg) / {0} i.e., indices of pixels in SEG which constitute a region mask ninstance : no of times image should be used to place text. """ for i in range(ninstance): place_masks = place_mask.copy() idict = {'img':[], 'charBB':None, 'wordBB':None, 'txt':None, 'lineBB':None} placed = False img = rgb.copy() itext = [] ibb = [] for idx in range(self.NUM_REP): try: txt_render_res = self.place_text(img, place_masks) except: traceback.print_exc() # some error in placing text on the region continue if txt_render_res is not None: placed = True img,text,bb,collision_mask = txt_render_res # update the region collision mask: place_masks = collision_mask # store the result: itext.append(text) ibb.append(bb) if placed: # at least 1 word was placed in this instance: idict['charBB'] = np.concatenate(ibb, axis=2) idict['lineBB'], idict['txt'], letterBB = self.char2lineBB(idict['charBB'].copy(), itext, place_masks.shape) outimg = Image.fromarray(img) inimgname_prefix = os.path.splitext(dir['inimgname'])[0] outimg.save(os.path.join(dir['outimgdir'],inimgname_prefix +'_'+str(i).zfill(3)+'.png'),'png') save_label_txt(os.path.join(dir['outtxtdir'],inimgname_prefix +'_'+str(i).zfill(3)+'.txt'), idict['lineBB'], idict['txt'], letterBB) if viz: viz_textbb(1,img, [idict['lineBB']], alpha=1.0) # viz_masks(2,img,seg,depth,regions['label']) # viz_regions(rgb.copy(),xyz,seg,regions['coeff'],regions['label']) if i < ninstance-1: raw_input(colorize(Color.BLUE,'continue?',True)) else: if global_util.logfile is not None: global_util.logfile.write("{} failed\n".format(os.path.join(dir['outimgdir'],dir['inimgname'])))
class RendererV3(object): def __init__(self, data_dir, max_time=None): self.text_renderer = tu.RenderFont(data_dir) self.colorizer = Colorize(data_dir) #self.colorizerV2 = colorV2.Colorize(data_dir) self.min_char_height = 8#8 #px self.min_asp_ratio = 0.4#0.4 # self.max_text_regions = 1#Rohit changed from 7 to 1 self.max_time = max_time def filter_regions(self,regions,filt): """ filt : boolean list of regions to keep. """ idx = np.arange(len(filt))[filt] for k in regions.keys(): regions[k] = [regions[k][i] for i in idx] return regions def filter_for_placement(self,xyz,seg,regions): filt = np.zeros(len(regions['label'])).astype('bool') masks,Hs,Hinvs = [],[], [] for idx,l in enumerate(regions['label']): res = get_text_placement_mask(xyz,seg==l,regions['coeff'][idx],pad=2) if res is not None: mask,H,Hinv = res masks.append(mask) Hs.append(H) Hinvs.append(Hinv) filt[idx] = True regions = self.filter_regions(regions,filt) regions['place_mask'] = masks regions['homography'] = Hs regions['homography_inv'] = Hinvs return regions def warpHomography(self,src_mat,H,dst_size): dst_mat = cv2.warpPerspective(src_mat, H, dst_size, flags=cv2.WARP_INVERSE_MAP|cv2.INTER_LINEAR) return dst_mat def homographyBB(self, bbs, H, offset=None): """ Apply homography transform to bounding-boxes. BBS: 2 x 4 x n matrix (2 coordinates, 4 points, n bbs). Returns the transformed 2x4xn bb-array. offset : a 2-tuple (dx,dy), added to points before transfomation. """ eps = 1e-16 # check the shape of the BB array: t,f,n = bbs.shape assert (t==2) and (f==4) # append 1 for homogenous coordinates: bbs_h = np.reshape(np.r_[bbs, np.ones((1,4,n))], (3,4*n), order='F') if offset != None: bbs_h[:2,:] += np.array(offset)[:,None] # perpective: bbs_h = H.dot(bbs_h) bbs_h /= (bbs_h[2,:]+eps) bbs_h = np.reshape(bbs_h, (3,4,n), order='F') return bbs_h[:2,:,:] def bb_filter(self,bb0,bb,text): """ Ensure that bounding-boxes are not too distorted after perspective distortion. bb0 : 2x4xn martrix of BB coordinates before perspective bb : 2x4xn matrix of BB after perspective text: string of text -- for excluding symbols/punctuations. """ # return True h0 = np.linalg.norm(bb0[:,3,:] - bb0[:,0,:], axis=0) w0 = np.linalg.norm(bb0[:,1,:] - bb0[:,0,:], axis=0) hw0 = np.c_[h0,w0] h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) w = np.linalg.norm(bb[:,1,:] - bb[:,0,:], axis=0) hw = np.c_[h,w] if np.sum(hw0[:,1] == 0) > 0 or np.sum(hw[:,1] == 0) > 0: return False # remove newlines and spaces: # text = ''.join(text.split()) # assert len(text)==bb.shape[-1] # alnum = np.array([ch.isalnum() for ch in text]) # hw0 = hw0[alnum,:] # hw = hw[alnum,:] min_h0, min_h = np.min(hw0[:,0]), np.min(hw[:,0]) asp0, asp = hw0[:,0]/hw0[:,1], hw[:,0]/hw[:,1] asp0, asp = np.median(asp0), np.median(asp) asp_ratio = asp/asp0 is_good = ( min_h > self.min_char_height and asp_ratio > self.min_asp_ratio and asp_ratio < 1.0/self.min_asp_ratio) return is_good def get_min_h(self, bb, text): # return True # find min-height: h = np.linalg.norm(bb[:,3,:] - bb[:,0,:], axis=0) # remove newlines and spaces: text = ''.join(text.split()) # assert len(text)==bb.shape[-1] # alnum = np.array([ch.isalnum() for ch in text]) # h = h[alnum] return np.min(h) def feather(self, text_mask, min_h): # determine the gaussian-blur std: bsz = 0.5 * np.random.rand() ksz = np.random.choice([1,3,5]) # if min_h <= 15 : # bsz = 0.25 # ksz=1 # elif 15 < min_h < 30: # bsz = max(0.30, 0.5 + 0.1*np.random.randn()) # ksz = 3 # else: # bsz = max(0.5, 1.5 + 0.5*np.random.randn()) # ksz = 5 return cv2.GaussianBlur(text_mask,(ksz,ksz),bsz) def place_text(self,rgb,collision_mask,H,Hinv): font = self.text_renderer.font_state.sample() font = self.text_renderer.font_state.init_font(font) # print("syngthgen #510") render_res = self.text_renderer.render_sample(font,collision_mask) # print("syngthgen #513") if render_res is None: # rendering not successful return #None else: text_mask,loc,bb,text = render_res # update the collision mask with text: collision_mask += (255 * (text_mask>0)).astype('uint8') # warp the object mask back onto the image: text_mask_orig = text_mask.copy() bb_orig = bb.copy() text_mask = self.warpHomography(text_mask,H,rgb.shape[:2][::-1]) bb = self.homographyBB(bb,Hinv) if not self.bb_filter(bb_orig,bb,text): #warn("bad charBB statistics") return #None # get the minimum height of the character-BB: min_h = self.get_min_h(bb,text) #feathering: text_mask = self.feather(text_mask, min_h) im_final = self.colorizer.color(rgb,[text_mask],np.array([min_h])) return im_final, text, bb, collision_mask def get_num_text_regions(self, nregions): #return nregions nmax = min(self.max_text_regions, nregions) if np.random.rand() < 0.10: rnd = np.random.rand() else: rnd = np.random.beta(5.0,1.0) return 1#int(np.ceil(nmax * rnd))#ROHIT CHANGED THIS def check_word_scriptDev(self, word_text): #devchars= 'ाीिुूेैंोौृःॄॅअआइईउऊऋएऐओऔअंअःकखगघङचछजझञटठडढणतथदधनपफबभमयरलवषशसहक्षत्रज्ञड़ढ़।॥' devFlag = 0 #print(word_text) for chrs in word_text: if(chrs in self.devchars): devFlag = 1 break; return devFlag def aksharaSplit(self, word_text): prevWasHalanta = False word_text_new = [] for chr1 in word_text: if chr1 in 'ाीिुूेैंोौृःॄॅ': prev_char = word_text_new[-1] word_text_new.pop() word_text_new.append(prev_char + chr1) prevWasHalanta = False elif chr1 in "्": prevWasHalanta = True else: if(prevWasHalanta): prev_char = word_text_new[-1] word_text_new.pop() word_text_new.append(prev_char + "्" + chr1) else: word_text_new.append(chr1) prevWasHalanta = False return word_text_new def char2wordBB(self, charBB, text): """ Converts character bounding-boxes to word-level bounding-boxes. charBB : 2x4xn matrix of BB coordinates text : the text string output : 2x4xm matrix of BB coordinates, where, m == number of words. """ wrds = text.split() bb_idx = np.r_[0, np.cumsum([len(self.aksharaSplit(w)) for w in wrds])] wordBB = np.zeros((2,4,len(wrds)), 'float32') #print("wrds", wrds) for i in range(len(wrds)): cc = charBB[:,:,bb_idx[i]:bb_idx[i+1]] # fit a rotated-rectangle: # change shape from 2x4xn_i -> (4*n_i)x2 cc = np.squeeze(np.concatenate(np.dsplit(cc,cc.shape[-1]),axis=1)).T.astype('float32') rect = cv2.minAreaRect(cc.copy()) box = np.array(cv2.boxPoints(rect)) # find the permutation of box-coordinates which # are "aligned" appropriately with the character-bb. # (exhaustive search over all possible assignments): cc_tblr = np.c_[cc[0,:], cc[-3,:], cc[-2,:], cc[3,:]].T perm4 = np.array(list(itertools.permutations(np.arange(4)))) dists = [] for pidx in range(perm4.shape[0]): d = np.sum(np.linalg.norm(box[perm4[pidx],:]-cc_tblr,axis=1)) dists.append(d) wordBB[:,:,i] = box[perm4[np.argmin(dists)],:].T return wordBB def word2lineBB(self, wordBB, text): """ Converts character bounding-boxes to word-level bounding-boxes. wordBB : 2x4xn matrix of BB coordinates text : the word text string output : 2x4xm matrix of BB coordinates, where, m == number of lines. """ print("text", text) lines = text.split("\n") bb_idx = np.r_[0, np.cumsum([len(l.split()) for l in lines])] lineBB = np.zeros((2,4,len(lines)), 'float32') print("lines", lines) for i in range(len(lines)): cc = wordBB[:,:,bb_idx[i]:bb_idx[i+1]] # fit a rotated-rectangle: # change shape from 2x4xn_i -> (4*n_i)x2 cc = np.squeeze(np.concatenate(np.dsplit(cc,cc.shape[-1]),axis=1)).T.astype('float32') rect = cv2.minAreaRect(cc.copy()) box = np.array(cv2.boxPoints(rect)) # find the permutation of box-coordinates which # are "aligned" appropriately with the character-bb. # (exhaustive search over all possible assignments): cc_tblr = np.c_[cc[0,:], cc[-3,:], cc[-2,:], cc[3,:]].T perm4 = np.array(list(itertools.permutations(np.arange(4)))) dists = [] for pidx in range(perm4.shape[0]): d = np.sum(np.linalg.norm(box[perm4[pidx],:]-cc_tblr,axis=1)) dists.append(d) lineBB[:,:,i] = box[perm4[np.argmin(dists)],:].T return lineBB '''# wrds = text.split() # bb_idx = np.r_[0, np.cumsum([len(w) for w in wrds])] # wordBB = np.zeros((2,4,len(wrds)), 'float32') # for i in range(len(wrds)): # cc = charBB[:,:,bb_idx[i]:bb_idx[i+1]] # # fit a rotated-rectangle: # # change shape from 2x4xn_i -> (4*n_i)x2 # cc = np.squeeze(np.concatenate(np.dsplit(cc,cc.shape[-1]),axis=1)).T.astype('float32') # rect = cv2.minAreaRect(cc.copy()) # box = np.array(cv2.boxPoints(rect)) # # find the permutation of box-coordinates which # # are "aligned" appropriately with the character-bb. # # (exhaustive search over all possible assignments): # cc_tblr = np.c_[cc[0,:], # cc[-3,:], # cc[-2,:], # cc[3,:]].T # perm4 = np.array(list(itertools.permutations(np.arange(4)))) # dists = [] # for pidx in range(perm4.shape[0]): # d = np.sum(np.linalg.norm(box[perm4[pidx],:]-cc_tblr,axis=1)) # dists.append(d) # wordBB[:,:,i] = box[perm4[np.argmin(dists)],:].T return charBB''' def render_text(self,rgb,depth,seg,area,label,ninstance=1,viz=False):# takes rgb image, depth, segmentation masks seg, label indices of pixel mask, ninstance, no of times image should be used to place text and returns a list of dictionaries one for each image instance with img bbs and txt, ith non-space white char in txt is bb[:,:,i] """ rgb : HxWx3 image rgb values (uint8) depth : HxW depth values (float) seg : HxW segmentation region masks area : number of pixels in each region label : region labels == unique(seg) / {0} i.e., indices of pixels in SEG which constitute a region mask ninstance : no of times image should be used to place text. @return: res : a list of dictionaries, one for each of the image instances. Each dictionary has the following structure: 'img' : rgb-image with text on it. 'bb' : 2x4xn matrix of bounding-boxes for each character in the image. 'txt' : a list of strings. The correspondence b/w bb and txt is that i-th non-space white-character in txt is at bb[:,:,i]. If there's an error in pre-text placement, for e.g. if there's no suitable region for text placement, an empty list is returned. """ try: # depth -> xyz xyz = su.DepthCamera.depth2xyz(depth) # find text-regions: regions = TextRegions.get_regions(xyz,seg,area,label)# taking depth, segmentations, area and labels and return regions #print("herE", regions) #regions = [regions[0]] # find the placement mask and homographies: regions = self.filter_for_placement(xyz,seg,regions) # finally place some text: nregions = len(regions['place_mask']) if nregions < 1: # no good region to place text on return [] except: # failure in pre-text placement #import traceback traceback.print_exc() return [] res = [] for i in range(ninstance): place_masks = copy.deepcopy(regions['place_mask']) # print (colorize(Color.CYAN, " ** instance # : %d"%i)) idict = {'img':[], 'charBB':None, 'wordBB':None, 'txt':None} m = self.get_num_text_regions(nregions)#np.arange(nregions)#min(nregions, 5*ninstance*self.max_text_regions)) reg_idx = np.arange(min(1,nregions))#Rohit changed 2*m to 1 np.random.shuffle(reg_idx) reg_idx = reg_idx[:m] placed = False img = rgb.copy() itext = [] ibb = [] #print (colorize(Color.CYAN, " ** 2instance # : %d"%i)) # process regions: num_txt_regions = len(reg_idx) NUM_REP = 1# ROHIT CHANGED TO 2 FROM 5 # re-use each region three times: reg_range = np.arange(NUM_REP * num_txt_regions) % num_txt_regions # print("reg_range is ", reg_range) for idx in reg_range: ireg = reg_idx[idx] try: if self.max_time is None: txt_render_res = self.place_text(img,place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) else: with time_limit(self.max_time):#SECS_PER_IMG = 5 #max time per image in seconds txt_render_res = self.place_text(img,place_masks[ireg], regions['homography'][ireg], regions['homography_inv'][ireg]) except TimeoutException as msg: print (msg) continue except: traceback.print_exc() # some error in placing text on the region continue if txt_render_res is not None: placed = True img,text,bb,collision_mask = txt_render_res # update the region collision mask: place_masks[ireg] = collision_mask # store the result: itext.append(text) ibb.append(bb) if placed: # at least 1 word was placed in this instance: idict['img'] = img idict['txt'] = itext #print("itext", itext) idict['charBB'] = np.concatenate(ibb, axis=2) idict['wordBB'] = self.char2wordBB(idict['charBB'].copy(), ' '.join(itext)) idict['lineBB'] = self.word2lineBB(idict['wordBB'].copy(), ' '.join(itext)) res.append(idict.copy()) if viz: viz_textbbchar(1,img, [idict['charBB']], alpha=1.0) viz_textbb(1,img, [idict['wordBB']], alpha=1.0) viz_textbb(1,img, [idict['lineBB']], alpha=1.0) viz_masks(2,img,seg,depth,regions['label']) # viz_regions(rgb.copy(),xyz,seg,regions['coeff'],regions['label']) if i < ninstance-1: input(colorize(Color.BLUE,'continue?',True)) return res
if __name__ == '__main__': # testing: Renderer = WordRenderer(is_debug=True, Language="Vietnamese") for i in (range(40)): print(f"{i}--------") # Img[HxWx4], Texts(List[str]), CBOX[nx4x2], BBOX[nx4x2] (N_box, corner, x/y) bg_img = cv2.imread("img_2.jpg") img, texts, cbox, bbox = Renderer.RenderWordImageA( 660, 1000, (30, 30, 30, 240)) text_mask = img[:, :, 3] print(text_mask.shape) print(bg_img.shape) bg_img = cv2.resize(bg_img, (text_mask.shape[1], text_mask.shape[0])) Colorizer = Colorize() img = Colorizer.color(bg_img, np.array([text_mask]), np.array([30])) print(texts) # print(img.shape) # visualize boxes img_b = img.copy() for box in bbox: img_b[box[0, 1]:box[1, 1] + 2, box[0, 0]:box[1, 0] + 2] = [255, 0, 0, 255] img_b[box[2, 1]:box[3, 1] + 2, box[3, 0]:box[2, 0] + 2] = [0, 255, 0, 255] img_b[box[0, 1]:box[3, 1] + 2, box[0, 0]:box[3, 0] + 2] = [255, 0, 255, 255] img_b[box[1, 1]:box[2, 1] + 2, box[1, 0]:box[2, 0] + 2] = [0, 255, 255, 255] img_c = img.copy()