def getSmallestBoundRect(mRectBoundOfTexts, rect): smalledRect = Rect() # check if there is any rectangle contains with this rect for erRect in mRectBoundOfTexts: if (RectUtil.contains(erRect, rect)): if (smalledRect.area() == 0): smalledRect = erRect elif (RectUtil.contains(smalledRect, erRect)): smalledRect = erRect return smalledRect
def validateWordWithAllViews(self, tv, ocrTextWrapper): ocrBound = ocrTextWrapper.bound() for view in self.mViews: # woa this word is big and have a lot of children, not good # this may okay with url or special texts if (RectUtil.contains(ocrBound, view.bound()) and len(view.getChildren()) > 0): tv.scalar = ColorUtil.getScalar(CColor.Cyan) tv.valid = False tv.log = "This word is big and have a lot of children" return # the box may has the word is too small compare with the word # itself. # If the word a children view which only have on child, we need # to verify if: # (1) This child view did not intersect with any other views # (2) This child view really small compare to the word bound if RectUtil.contains(ocrBound, view.bound()) and len( view.getChildren()) == 0: # make sure this view did not intersect with other view, # include is accepted in this case hasIntersection = False for otherView in self.mViews: if otherView != view and RectUtil.intersectsNotInclude( ocrBound, otherView.bound()): hasIntersection = True break if (not hasIntersection): if (RectUtil.dimesionSmallerThan(view, ocrTextWrapper, 0.8)): # this is wrong, ignore this word tv.scalar = CColor.Black tv.valid = False tv.log = "The box may has the word is too small compare with the word itself" return # same here: the box may has the word is too big compare to the # word itself we also make sure that this view may also have # other view but it so tiny will be ignore when layout # itself and there is only one word in here if (self.areChildrenIsTooSmall(self.mDipCalculator, view) and RectUtil.contains(view.bound(), ocrBound)): if (RectUtil.dimesionSmallerThan(ocrTextWrapper, view, 0.8)): # this is wrong, ignore this word tv.scalar = CColor.Pink tv.valid = False tv.log = "The box may has the word is too big compare to the word, and there is only one word in here. This view may also have other view but it so tiny" return
def accept(self, ocr): # return None bound = ocr.bound() for view in self.mViews: # the box may has the word is too small compare with the word # itself. # If the word a children view which only have on child, we need # to verify if: # (1) This child view did not intersect with any other views if RectUtil.contains(bound, view.bound()) and len( view.mChildren) == 0: # make sure this view did not intersect with other view, # include is accepted in this case hasIntersection = False for otherView in self.mViews: if otherView != view and RectUtil.intersectsNotInclude( bound, otherView.bound()): hasIntersection = True break if not hasIntersection: if RectUtil.dimesionSmallerThan( view, ocr, 0.8): # this is wrong, ignore this word tv = TextValidator( ocr, CColor.Black, False, "The box may has the word is too small compare with the word itself" ) return tv return None
def addTextToHierarchyInternal(self, view, blocks): children = view.mChildren for childView in children: self.addTextToHierarchyInternal(childView, blocks) childBlocks = TextProcessorUtil.getTextAndRemove(view, blocks) if (len(childBlocks) == 0): return removedChildren = [] for rectView in children: removed = False for ocrBlock in childBlocks: # // if it overlap a view but not include in it, we should just # // remove these views # // if children already has text, we should keep them # // *** There is situation in which the text right at the edge so # // the vision box # // need to expand just a bit (1 px each dimension) to cover the # // text if (not RectUtil.contains(RectUtil.expand1Px(rectView.bound()), ocrBlock.bound()) and RectUtil.intersects(ocrBlock.bound(), rectView.bound()) and not rectView.hasTextRecusive()): removed = True break if (removed): removedChildren.append(rectView) view.mChildren = [ x for x in view.mChildren if x not in removedChildren ] view.mTextWithLocations = childBlocks view.mTextChildren = removedChildren # // We want to remove all grand children vision box too. # // This is the error we found in TimeHop iOS app. leafNodes = RectUtil.getLeafNodes(view) for rectViewPair in leafNodes: for text in childBlocks: if (RectUtil.contains(text, rectViewPair[0].rect, 0.75) and rectViewPair[0].mChildren is not None): rectViewPair[0].mChildren = rectViewPair[ 0].mChildren.remove(rectViewPair[1]) break
def getWordsIn(self, rect): wrappers = [] for ocrTextWrapper in self.mOcrTextWrappers: bound = ocrTextWrapper.bound() if (RectUtil.contains(rect, bound)): wrappers.append(OCRTextWrapper.OCRTextWrapper(ocrTextWrapper)) return wrappers
def getBlockWithLocation(self, rect): wrappers = [] for ocrTextWrapper in self.mOcrBlockWrappers: bound = ocrTextWrapper.rect if (RectUtil.contains(rect, bound)): wrappers.append(OCRTextWrapper.OCRTextWrapper(ocrTextWrapper)) return wrappers
def getTextAndRemove(viewBound, blocks): childTexts = [] for ocrTextWrapper in blocks: bound = ocrTextWrapper.bound() if (RectUtil.contains(viewBound, bound)): childTexts.append(ocrTextWrapper) blocks = [x for x in blocks if x not in childTexts] return childTexts
def hasParent(ocrTextWrapper, ocrWrappers): # keep word did not have children hasParent = False for otherOcrTextWrapper in ocrWrappers: if otherOcrTextWrapper != ocrTextWrapper and RectUtil.contains( otherOcrTextWrapper.rect, ocrTextWrapper.rect): return True return hasParent
def accept(self,ocr): bound = ocr.bound() # return None for view in self.mViews: # woa this word is big and have a lot of children, not good # this may okay with url or special texts if RectUtil.contains(bound, view.bound()) and len(view.mChildren) > 0: tv = TextValidator(ocr, CColor.Cyan, False, "This word is big and have a lot of children" + str(len(view.mChildren))) return tv return None
def initLine(self): self.mOcrLineWrappers = self.baseInit(RIL.TEXTLINE) invalidLineWrappers = [] # a line cannot contain another lines for ocrLine in self.mOcrLineWrappers: for otherOcrLine in self.mOcrLineWrappers: if (ocrLine != otherOcrLine and RectUtil.contains( ocrLine.bound(), otherOcrLine.bound())): invalidLineWrappers.append(ocrLine) self.mOcrLineWrappers = [ x for x in self.mOcrLineWrappers if x not in invalidLineWrappers ]
def run(self, invalidTexts, acceptedOcrTextWrappers): # Step 2: Validate base on view match = False for view in self.mViews: # the box may has the word is too big compare to the word # itself and there is only one word in here wordsIn = self.mTesseractOCR.getWordsIn(view.bound()) for key in invalidTexts: if key in wordsIn: wordsIn.remove(key) if len(wordsIn) == 1 and len(view.mChildren) == 0: # TODO: Try to find if there is any other word intersect with # this view if (RectUtil.contains(view, wordsIn[0])): ocrTextWrapper = wordsIn[0] ob = ocrTextWrapper.bound() vb = view.bound() ratioWidth = float(ob.width / vb.width) ratioHeight = float(ob.height / vb.height) maxDimRatio = max(ratioWidth, ratioHeight) minDimRatio = min(ratioWidth, ratioHeight) # 1. Both dim really small < 0.4F # 2. Max dim is a little bit small < 0.7F, but min dim is # small < 0.2F. # 3. Max dim is big, min dim have to be really small < 0.2F if (maxDimRatio < 0.4 or (maxDimRatio < 0.7 and minDimRatio < 0.4)) or ( maxDimRatio >= 0.7 and minDimRatio < 0.2): # this is wrong, ignore this word textValidator = TextValidator( ocrTextWrapper, CColor.Green, False, "The box, which may has the word, is too big compare to the word, and there is only one word in here" ) invalidTexts[textValidator.textWrapper] = textValidator match = True return match
def accept(self, ocr): rects = [] ocrBound = ocr.bound() for view in self.mViews: # accept overlap here if not self.mDipCalculator.isViewToBeIgnoreView( view) and RectUtil.contains(ocrBound, view.bound(), 0.5): rects.append(view.bound()) if len(rects) >= 2: RectUtil.sortTopBottom(rects) # distance between 2 characters are greater than the character # itself this have to be true for all of character invalidWordWithChars = False for i in range(len(rects) - 1): current = rects[i] next_rect = rects[i + 1] # ignore intersect one if RectUtil.intersects(current, next_rect): continue if current.width < next_rect.x - (current.x + current.width): invalidWordWithChars = True else: invalidWordWithChars = False break if invalidWordWithChars: # SkyBlue tv = TextValidator.TextValidator( ocr, (135, 206, 235), False, "distance between 2 characters are greater than the character itself: " ) return tv return None
def accept(self, ocr): bound = ocr.bound() for view in self.mViews: # // the box may has the word is too small compare with the word # // itself. # // If the word a children view which only have on child, we need # // to verify if: # // (2) This child view really small compare to the word bound # // same here: the box may has the word is too big compare to the # // word itself we also make sure that this view may also have # // other view but it so tiny will be ignore when layout # // itself and there is only one word in here if TextProcessorUtil.areChildrenIsTooSmall( self.mDipCalculator, view) and RectUtil.contains( view.bound(), bound): if RectUtil.dimesionSmallerThan(ocr, view, 0.8): # this is wrong, ignore this word tv = TextValidator( ocr, CColor.Pink, False, "The box may has the word is too big compare to the word, and there is only one word in here. This view may also have other view but it so tiny" ) return tv return None
def processText(self, color): ocrTextWrappers = self.mOcr.mOcrTextWrappers width = 0 height = 0 copyImage = copy.deepcopy(self.mRgbaImage) if len(copyImage.shape) == 2: height, width = copyImage.shape else: height, width, channels = copyImage.shape # ocrOnlyProcessingStepImage = copy.deepcopy(self.mRgbaImage) acceptedOcrTextWrappers = [] ruleManager = FilterRuleManager(self.mDipCalculator, self.mOcr, self.mRgbaImage, ocrTextWrappers, self.mViews) invalidTexts = {} for ocrTextWrapper in ocrTextWrappers: textValidator = ruleManager.acceptOCRRules(ocrTextWrapper) if textValidator != None and not textValidator.valid: invalidTexts[ocrTextWrapper] = textValidator # if(self.isDebugMode) : # cv2.rectangle(ocrOnlyProcessingStepImage, ocrTextWrapper.bound().tl(), ocrTextWrapper.bound().br(), CColor.Red, 2) else: acceptedOcrTextWrappers.append(ocrTextWrapper) # if(self.isDebugMode) : # cv2.rectangle(ocrOnlyProcessingStepImage, ocrTextWrapper.bound().tl(), ocrTextWrapper.bound().br(), CColor.Blue, 2) ruleManager.acceptVisionRules(invalidTexts, acceptedOcrTextWrappers) validTexts = [] validTexts.extend(acceptedOcrTextWrappers) validTexts = [x for x in validTexts if x not in invalidTexts] # ImageUtil.drawWindow( "basic Text",ocrOnlyProcessingStepImage) ocrLineWrappers = self.mOcr.mOcrLineWrappers # sort top bottom copyLines = [] copyLines.extend(ocrLineWrappers) copyLines.sort(key=cmp_to_key(RectUtil.getTopBottomComparator)) validLines = [] addedWords = [] for ocrLineWrapper in copyLines: words = [] line = OCRTextWrapper.OCRTextWrapper(ocrLineWrapper) for ocrWordWrapper in validTexts: if (ocrWordWrapper not in addedWords) and RectUtil.contains( ocrLineWrapper.bound(), ocrWordWrapper.bound()): words.append(ocrWordWrapper) addedWords.append(ocrWordWrapper) # Some line contain 2 words which are vertically alignment if len(words) > 0: notHorizontalAlignmentWords = self.getNotHorizontalAlignmentWords( words) if len(notHorizontalAlignmentWords) == 0: validLines.append(line) words.sort(key=cmp_to_key(RectUtil.getLeftRightComparator)) line.words = words else: # Take it from addedWords. This will help these words be # added to other lines, since this line is invalid addedWords = [ x for x in addedWords if x not in notHorizontalAlignmentWords ] # remove bad guy words = [ x for x in words if x not in notHorizontalAlignmentWords ] validLines.append(line) words.sort(key=cmp_to_key(RectUtil.getLeftRightComparator)) line.words = words # We still want to add word as line when it did not get add to any # lines remainWords = [] remainWords.extend(validTexts) remainWords = [x for x in remainWords if x not in addedWords] for word in remainWords: # System.out.println("Remain words: " + word); if (word.confidence < 90 and not self.mOcr.isValidTextUsingBoundaryCheck(word)): continue line = OCRTextWrapper.OCRTextWrapper(word) words = [] words.append(word) line.words = words validLines.append(line) validLines.sort(key=cmp_to_key(RectUtil.getTopBottomComparator)) # self.log("ValidLines", validLines, CColor.Red) for ocrLineWrapper in validLines: rect = ocrLineWrapper.reCalculateBoundBaseOnWordList() if (rect == None): print("Error with line, there is no more text: " + ocrLineWrapper.text) #System.out.println("Error with line, there is no more text: "+ ocrLineWrapper.getText()); else: text = self.mOcr.getText(rect) ocrLineWrapper.text = text ocrLineWrapper.rect = rect # word is sort from left to right for ocrLineWrapper in validLines: blocks = [[]] words = ocrLineWrapper.words currentBlock = [] if len(words) > 0: currentBlock.append(words[0]) for i in range(len(words) - 1): nextWord = words[i + 1] currentWord = words[i] xDistance = nextWord.x - (currentWord.x + currentWord.width) xDistanceThreshold = int( Constants.WORD_SPACE_THRESHOLD_BASE_ON_HEIGHT * float( min(currentWord.bound().height, nextWord.bound().height))) fontDiff = abs(currentWord.fontSize - nextWord.fontSize) # if (xDistance <= xDistanceThreshold and fontDiff <= 1) : if (xDistance <= xDistanceThreshold): currentBlock.append(nextWord) else: blocks.append(currentBlock) currentBlock = [] currentBlock.append(nextWord) if currentBlock not in blocks: blocks.append(currentBlock) ocrLineWrapper.blocks = blocks # logImageWithValidTextBox = copy.deepcopy(self.mRgbaImage) # ImageUtil.fillRect(logImageWithValidTextBox, Rect(0, 0, width,height), ColorUtil.toInt(255, 255, 255, 255)) blocksInline = [] for lineOCR in validLines: blocks = lineOCR.blocks for listWord in blocks: if len(listWord) > 0: firstWord = listWord[0] rect = RectUtil.findBoundRectangle(listWord) # cv2.rectangle(logImageWithValidTextBox, rect.tl(), # rect.br(), CColor.Red, 2); block = OCRTextWrapper.OCRTextWrapper(firstWord) block.words = listWord block.width = rect.width block.height = rect.height block.rect = rect lineText = "" # rect = Rect(rect.x -2, rect.y-2, rect.width +2, rect.height +2) # if len(listWord) == 1 : # lineText = listWord[0].text # else : # # override text lineText = self.mOcr.getLineText(rect) block.text = lineText # will ignore this block if it contains only invisible # chars # blocksInline.append(block) if (RuleAllSpace.containAllSpacesOrInvalidChars(lineText)): blocksInline.append(block) colListmap = {} for blocks in blocksInline: colListmap[ColorWrapper(ColorUtil.cColortoInt(CColor.Red), 1)] = blocks # ImageUtil.logDrawMap(colListmap, "Text Block", self.mRgbaImage) textInfo = TextInfo() textInfo.lines = validLines textInfo.blocksInALine = blocksInline textInfo.blocksInALine.sort( key=cmp_to_key(RectUtil.getTopBottomComparator)) #mSreenshotProcessor.getTimerManager().log(Constants.TIMER_ID_SPLIT_LINE_INTO_TEXT_BOXES); return textInfo
def reorganizeParentChildHierachyInternal(self, view): children = view.mChildren for childView in view.mChildren: self.reorganizeParentChildHierachyInternal(childView) hierachy = {} len(children) for i in range(len(children)): rectView = children[i] for j in range(i + 1, len(children)): otherRectView = children[j] if (rectView != otherRectView): if (RectUtil.contains(rectView.bound(), otherRectView.bound())): if j in hierachy and i in hierachy[j]: #"This is wrong: " + j+ " already is parents of " + i pass else: childrenIndexes = [] if (not i in hierachy): childrenIndexes = [] hierachy[i] = childrenIndexes else: childrenIndexes = hierachy[i] if (not j in childrenIndexes): childrenIndexes.append(j) elif (RectUtil.contains(otherRectView.bound(), rectView.bound())): if i in hierachy and j in hierachy[i]: #"This is wrong: " + i+ " already is parents of " + j); pass else: childrenIndexes = [] if (not j in hierachy): childrenIndexes = [] hierachy[j] = childrenIndexes else: childrenIndexes = hierachy[j] if (not i in childrenIndexes): childrenIndexes.append(i) # remove view if it children claim that its contain that view optinmizedHierarchy = {} parentIndexes = [] for i in hierachy: childrenIndexes = hierachy[i] newChildrenIndexes = [] for childrenIndex in childrenIndexes: # remove grandchildren belong to its children if childrenIndex in hierachy: grandChildrenIndexes = hierachy[childrenIndex] newChildrenIndexes = [ x for x in newChildrenIndexes if x not in grandChildrenIndexes ] optinmizedHierarchy[i] = newChildrenIndexes parentIndexes.append(i) # remove duplication of children includedChildrenIndexes = set() for parentIndex in parentIndexes: childrenIndexes = optinmizedHierarchy[parentIndex] includedChildrenIndexes.update(childrenIndexes) childrenIndexes.clear() childrenIndexes.extend(includedChildrenIndexes) # now we reordering the hierarchy for i in optinmizedHierarchy: childrenIndexes = optinmizedHierarchy[i] childrenIndexes.sort() rawView = children[i] for childIndex in childrenIndexes: rawView.addChild(children[childIndex]) removingChildrenIndexes = [False] * len(children) # all children value of this hierarchy map will be removed for i in optinmizedHierarchy: childrenIndexes = optinmizedHierarchy[i] for childIndex in childrenIndexes: removingChildrenIndexes[childrenIndex] = True # now we add parent according to order rawViews = [] indexes = [] for i in range(len(removingChildrenIndexes)): if (not removingChildrenIndexes[i]): indexes.append(i) indexes.sort() for index in indexes: rawViews.append(children[index]) view.mChildren = [x for x in children if x in rawViews]
def delOverlapViews(self, view): overlapIndexes = [] children = view.mChildren for rawView in children: self.delOverlapViews(rawView) removingChildrenIndexes = [False] * len(children) for i in range(len(children)): rectView = children[i] for j in range(i + 1, len(children)): otherRectView = children[j] if rectView != otherRectView: #Check if Intersects only if (not RectUtil.contains(rectView.bound(), otherRectView.bound()) and not RectUtil.contains(otherRectView.bound(), rectView.bound()) and RectUtil.intersects(rectView.bound(), otherRectView.bound())): added = False #Check if already in the overlapIndexes for overlapIndex in overlapIndexes: if (i in overlapIndex and j in overlapIndex): added = True break elif i in overlapIndex: overlapIndex.append(j) added = True break elif j in overlapIndex: overlapIndex.append(i) added = True break # did not contain non of them them add two of them as a list within overlapIndexes if (not added): rectViews = [] rectViews.append(i) rectViews.append(j) overlapIndexes.append(rectViews) removingChildrenIndexes[i] = True removingChildrenIndexes[j] = True newRectViews = [] # process overlap rect for rectViewIndexes in overlapIndexes: #find bound rectView = children[rectViewIndexes[0]] unionRect = rectView.bound() for i in range(1, len(rectViewIndexes)): r = children[rectViewIndexes[i]] unionRect = RectUtil.union(unionRect, r.bound()) newRectViewParent = RectView( Rect(unionRect.x, unionRect.y, unionRect.width, unionRect.height), None) for index in rectViewIndexes: newRectViewParent.addChild(children[index]) removingChildrenIndexes[rectViewIndexes[0]] = False view.mChildren[rectViewIndexes[0]] = newRectViewParent newRectViews.append(newRectViewParent) # now we make sure all other children of the parent inside union rect # go under this view too. for rectView in newRectViews: for i in range(len(removingChildrenIndexes)): rawView = children[i] if rawView != rectView and not removingChildrenIndexes[i]: bound = children[i].bound() if (RectUtil.contains(rectView.bound(), bound)): rectView.addChild(children[i]) removingChildrenIndexes[i] = True # now update the children rawViews = [] for i in range(len(removingChildrenIndexes)): if (not removingChildrenIndexes[i]): rawViews.append(children[i]) view.mChildren = [x for x in children if x in rawViews]
def prepareCreateListView(self, rectView, matchedBaseLists, alignmentType): largestSublists = set() # Remove lists which are subset of other lists for list1 in matchedBaseLists: # Remove sublists of list1 first subLists = set() for list2 in largestSublists: if all(x in list1._list for x in list2._list): subLists.add(list2) largestSublists.remove(subLists) # Will add list 1 to largestSublists if there is no list which # contains list1 shouldAddList1 = True for list2 in largestSublists: if all(x in list1._list for x in list2._list): shouldAddList1 = False if (shouldAddList1): largestSublists.add(list1) matchedBaseLists = [] matchedBaseLists.extend(largestSublists) # for ( ListWrapper matchedDrawableList : matchedBaseLists) : # System.out.prln("After still: " + rectView.bound() + ", list: " # + matchedDrawableList.size() + ", " + matchedDrawableList) # notDrwableRectViews = [] for _list in matchedBaseLists: notDrwableRectViews = [ x for x in notDrwableRectViews if x not in _list._list ] minSize = RectUtil.minSizeListWrapper(matchedBaseLists) # Now take min size as the base then group them groups = [] for i in range(len(minSize)): listItem = ListItemMetadata() for matchedDrawableList in matchedBaseLists: listItem.baseViews.append(matchedDrawableList_list[i]) # listItem.baseViews.size > 0 listItem.bound = RectUtil.findBoundRectangle(listItem.baseViews) groups.append(listItem) # Find views overlap these bound of each list add put them o # the group processedView = [] for listItem in groups: for notDrawableView in notDrwableRectViews: if (not processedView.contains(notDrawableView) and RectUtil.ersectsNotInclude( listItem.bound, notDrawableView.bound())): listItem.additionalViews.add(notDrawableView) processedView.add(notDrawableView) for listItem in groups: # We got a new bound here, this time more aggressive => we # only include view belong to # to the new bound if len(listItem.additionalViews) > 0: listItem.bound = RectUtil.union( listItem.bound, RectUtil.findBoundRectangle(listItem.additionalViews)) for notDrawableView in notDrwableRectViews: if (not processedView.contains(notDrawableView) and RectUtil.contains(listItem.bound, notDrawableView.bound())): listItem.additionalViews.add(notDrawableView) processedView.add(notDrawableView) # check if this list is the expand version of the previous list # find the first list which have same size expandingList = None for i in len(self.mListViews): listItemMetadatas = self.mListViews[i].mListItemMetadatas if len(listItemMetadatas) == len(groups): includeMetada = True for j in range(len(groups)): if (not (groups[j].baseViews.containsAll( listItemMetadatas[j].baseViews) and groups[j].baseViews.size() > listItemMetadatas[j].baseViews.size())): includeMetada = False if (includeMetada): expandingList = self.mListViews[i] break if (expandingList != None): # System.out.prln("Expanding: " + rectView) self.mListViews[self.mListViews.index( expandingList)] = ListMetadataRoot(rectView, groups, alignmentType) else: validList = True for listItem in groups: allViews = [] allViews.extend(listItem.additionalViews) bound = RectUtil.findBoundRectangle(allViews) countListView = 0 for listMetadata in self.mListViews: # contain entire a list is okay but only one list item # is not okay if (RectUtil.ersects(listMetadata.bound(), bound) or RectUtil.contains(listMetadata.bound(), bound)): countListView += countListView if (countListView >= 1): validList = False break if (validList): if len(groups) > 0: self.mListViews.append( ListMetadataRoot(rectView, groups, alignmentType)) invalidLists = [] for listMetadataRoot in self.mListViews: if (not self.validateList(listMetadataRoot)): invalidLists.append(listMetadataRoot) self.mListViews = [x for x in self.mListViews if x not in invalidLists]
def includes(self, bound): return RectUtil.contains(self.rect, bound)