def groupViewsForList(self, parentView, newChildren, viewTypeRect, minChidren): if len(newChildren) >= minChidren: bound = RectUtil.findBoundRectangle(newChildren) newParent = RectView(bound, None) newParent.mType = viewTypeRect # replace the parent at the location of the first child indexOf = parentView.mChildren.index(newChildren[0]) if indexOf > 0 and indexOf < len(parentView.mChildren): parentView.mChildren[indexOf] = newParent else: parentView.mChildren.append(newParent) # Now remove the rest parentView.mChildren = [ x for x in parentView.mChildren if x not in newChildren ] # Make sure there is no view is hidden under the new parent insideViews = RectUtil.contain(newParent, parentView.mChildren) parentView.mChildren = [ x for x in parentView.mChildren if x not in insideViews ] indexOfNewParent = parentView.mChildren.index(newParent) if (indexOfNewParent == len(parentView.mChildren) - 1): parentView.mChildren.extend(insideViews) else: parentView.mChildren.extend(indexOfNewParent + 1, insideViews) return newParent return None
def isValidList(self, viewList): if len(viewList) < Constants.LAYOUT_MIN_ACCEPTABLE_LIST_SIZE: return False bound = RectUtil.findBoundRectangle(viewList) heightDip = self.mDipCalculator.pxToHeightDip(bound.height) widthDip = self.mDipCalculator.pxToWidthDip(bound.width) area = heightDip * widthDip return area >= Constants.MIN_SINGLE_LIST_AREA
def processText(self, color): ocrTextWrappers = self.mOcr.mOcrTextWrappers width = 0 height = 0 copyImage = copy.deepcopy(self.mRgbaImage) if len(copyImage.shape) == 2: height, width = copyImage.shape else: height, width, channels = copyImage.shape # ocrOnlyProcessingStepImage = copy.deepcopy(self.mRgbaImage) acceptedOcrTextWrappers = [] ruleManager = FilterRuleManager(self.mDipCalculator, self.mOcr, self.mRgbaImage, ocrTextWrappers, self.mViews) invalidTexts = {} for ocrTextWrapper in ocrTextWrappers: textValidator = ruleManager.acceptOCRRules(ocrTextWrapper) if textValidator != None and not textValidator.valid: invalidTexts[ocrTextWrapper] = textValidator # if(self.isDebugMode) : # cv2.rectangle(ocrOnlyProcessingStepImage, ocrTextWrapper.bound().tl(), ocrTextWrapper.bound().br(), CColor.Red, 2) else: acceptedOcrTextWrappers.append(ocrTextWrapper) # if(self.isDebugMode) : # cv2.rectangle(ocrOnlyProcessingStepImage, ocrTextWrapper.bound().tl(), ocrTextWrapper.bound().br(), CColor.Blue, 2) ruleManager.acceptVisionRules(invalidTexts, acceptedOcrTextWrappers) validTexts = [] validTexts.extend(acceptedOcrTextWrappers) validTexts = [x for x in validTexts if x not in invalidTexts] # ImageUtil.drawWindow( "basic Text",ocrOnlyProcessingStepImage) ocrLineWrappers = self.mOcr.mOcrLineWrappers # sort top bottom copyLines = [] copyLines.extend(ocrLineWrappers) copyLines.sort(key=cmp_to_key(RectUtil.getTopBottomComparator)) validLines = [] addedWords = [] for ocrLineWrapper in copyLines: words = [] line = OCRTextWrapper.OCRTextWrapper(ocrLineWrapper) for ocrWordWrapper in validTexts: if (ocrWordWrapper not in addedWords) and RectUtil.contains( ocrLineWrapper.bound(), ocrWordWrapper.bound()): words.append(ocrWordWrapper) addedWords.append(ocrWordWrapper) # Some line contain 2 words which are vertically alignment if len(words) > 0: notHorizontalAlignmentWords = self.getNotHorizontalAlignmentWords( words) if len(notHorizontalAlignmentWords) == 0: validLines.append(line) words.sort(key=cmp_to_key(RectUtil.getLeftRightComparator)) line.words = words else: # Take it from addedWords. This will help these words be # added to other lines, since this line is invalid addedWords = [ x for x in addedWords if x not in notHorizontalAlignmentWords ] # remove bad guy words = [ x for x in words if x not in notHorizontalAlignmentWords ] validLines.append(line) words.sort(key=cmp_to_key(RectUtil.getLeftRightComparator)) line.words = words # We still want to add word as line when it did not get add to any # lines remainWords = [] remainWords.extend(validTexts) remainWords = [x for x in remainWords if x not in addedWords] for word in remainWords: # System.out.println("Remain words: " + word); if (word.confidence < 90 and not self.mOcr.isValidTextUsingBoundaryCheck(word)): continue line = OCRTextWrapper.OCRTextWrapper(word) words = [] words.append(word) line.words = words validLines.append(line) validLines.sort(key=cmp_to_key(RectUtil.getTopBottomComparator)) # self.log("ValidLines", validLines, CColor.Red) for ocrLineWrapper in validLines: rect = ocrLineWrapper.reCalculateBoundBaseOnWordList() if (rect == None): print("Error with line, there is no more text: " + ocrLineWrapper.text) #System.out.println("Error with line, there is no more text: "+ ocrLineWrapper.getText()); else: text = self.mOcr.getText(rect) ocrLineWrapper.text = text ocrLineWrapper.rect = rect # word is sort from left to right for ocrLineWrapper in validLines: blocks = [[]] words = ocrLineWrapper.words currentBlock = [] if len(words) > 0: currentBlock.append(words[0]) for i in range(len(words) - 1): nextWord = words[i + 1] currentWord = words[i] xDistance = nextWord.x - (currentWord.x + currentWord.width) xDistanceThreshold = int( Constants.WORD_SPACE_THRESHOLD_BASE_ON_HEIGHT * float( min(currentWord.bound().height, nextWord.bound().height))) fontDiff = abs(currentWord.fontSize - nextWord.fontSize) # if (xDistance <= xDistanceThreshold and fontDiff <= 1) : if (xDistance <= xDistanceThreshold): currentBlock.append(nextWord) else: blocks.append(currentBlock) currentBlock = [] currentBlock.append(nextWord) if currentBlock not in blocks: blocks.append(currentBlock) ocrLineWrapper.blocks = blocks # logImageWithValidTextBox = copy.deepcopy(self.mRgbaImage) # ImageUtil.fillRect(logImageWithValidTextBox, Rect(0, 0, width,height), ColorUtil.toInt(255, 255, 255, 255)) blocksInline = [] for lineOCR in validLines: blocks = lineOCR.blocks for listWord in blocks: if len(listWord) > 0: firstWord = listWord[0] rect = RectUtil.findBoundRectangle(listWord) # cv2.rectangle(logImageWithValidTextBox, rect.tl(), # rect.br(), CColor.Red, 2); block = OCRTextWrapper.OCRTextWrapper(firstWord) block.words = listWord block.width = rect.width block.height = rect.height block.rect = rect lineText = "" # rect = Rect(rect.x -2, rect.y-2, rect.width +2, rect.height +2) # if len(listWord) == 1 : # lineText = listWord[0].text # else : # # override text lineText = self.mOcr.getLineText(rect) block.text = lineText # will ignore this block if it contains only invisible # chars # blocksInline.append(block) if (RuleAllSpace.containAllSpacesOrInvalidChars(lineText)): blocksInline.append(block) colListmap = {} for blocks in blocksInline: colListmap[ColorWrapper(ColorUtil.cColortoInt(CColor.Red), 1)] = blocks # ImageUtil.logDrawMap(colListmap, "Text Block", self.mRgbaImage) textInfo = TextInfo() textInfo.lines = validLines textInfo.blocksInALine = blocksInline textInfo.blocksInALine.sort( key=cmp_to_key(RectUtil.getTopBottomComparator)) #mSreenshotProcessor.getTimerManager().log(Constants.TIMER_ID_SPLIT_LINE_INTO_TEXT_BOXES); return textInfo
def pruneBasicInternal(self, parent, view): # TODO: if this view is too small and it has no children,so we don't # need them if self.mDipCalculator.isViewToBeIgnore(view.width, view.height): if (parent != None): parent.mChildren.remove(view) return # # allChildrenAreTooSmall = self.isAllChildrenTooSmall(view) if not allChildrenAreTooSmall and len(view.mChildren) != 0: removedChildren = [] for childView in view.mChildren: if self.mDipCalculator.isViewToBeIgnore( childView.width, childView.height): removedChildren.append(childView) view.mChildren = [ x for x in view.mChildren if x not in removedChildren ] for childView in view.mChildren: self.pruneBasicInternal(view, childView) # # add this drawable if we did not want to show any children # # here # isAImageView = self.isFullImage(view) if isAImageView: # if not view.hasText() and allChildrenAreTooSmall and len(view.mChildren) == 0 : currentMat = ImageUtil.getImageFromRect(self.mImage, view.bound()) iconInfo = IconInfo(currentMat) drawableId = "" if iconInfo in self.interestedIcons: drawableId = self.interestedIcons[iconInfo] viewsSameDrawable = None if (TextUtils.isEmpty(drawableId)): drawableId = self.mDrawableWriter.addResourceDirectly( currentMat, view) self.interestedIcons[iconInfo] = drawableId viewsSameDrawable = [] self.mDrawableMap[drawableId] = viewsSameDrawable else: viewsSameDrawable = self.mDrawableMap[drawableId] view.mType = RectViewTypes.VIEW_TYPE_IMAGE view.mImageInfo.iconInfo = iconInfo view.mImageInfo.drawableId = drawableId view.mChildren = [] viewsSameDrawable.append(view) elif view.hasTextRecusive(): # process text view textWithLocations = view.mTextWithLocations view.mColor = ColorUtil.findDominateColor(view, self.mImage) for textWrapper in textWithLocations: newHeight = TesseractOCR.increaseHeight(textWrapper.height) textView = textWrapper.boundRectView newY = textView.y - (newHeight - textView.height) / 2.0 textView.y = newY textView.x = textWrapper.x textView.width = textWrapper.width textView.height = newHeight textView.mType = RectViewTypes.VIEW_TYPE_TEXT textView.mTextInfo.textWrapper = textWrapper textView.rect = Rect(textView.x, textView.y, textWrapper.width, textView.height) color = ColorUtil.findDominateColorForTextView( textView, self.mImage) textView.mColor = color[0] textView.textColor = color[1] # textView.mColor = ColorUtil.findDominateColor(textView,self.mImage) view.addChild(textView) # currentMat = ImageUtil.getImageFromRect(self.mImage, textView.bound()) # iconInfo = IconInfo(currentMat) # drawableId = "" # if iconInfo in self.interestedIcons: # drawableId = self.interestedIcons[iconInfo] # viewsSameDrawable = None # if (TextUtils.isEmpty(drawableId)) : # drawableId = self.mDrawableWriter.addResourceDirectly(currentMat,view) # self.interestedIcons[iconInfo] = drawableId # viewsSameDrawable = [] # self.mDrawableMap[drawableId] = viewsSameDrawable # else : # viewsSameDrawable = self.mDrawableMap[drawableId] # # textView.mType = RectViewTypes.VIEW_TYPE_IMAGE # textView.mImageInfo.iconInfo = iconInfo # textView.mImageInfo.drawableId = drawableId # textView.mChildren = [] # viewsSameDrawable.append(textView) # Update Bound of parent text view if len(view.mChildren) > 0: allViews = [] allViews.extend(view.mChildren) allViews.append(view) view.bound = RectUtil.findBoundRectangle(allViews)
def prepareCreateListView(self, rectView, matchedBaseLists, alignmentType): largestSublists = set() # Remove lists which are subset of other lists for list1 in matchedBaseLists: # Remove sublists of list1 first subLists = set() for list2 in largestSublists: if all(x in list1._list for x in list2._list): subLists.add(list2) largestSublists.remove(subLists) # Will add list 1 to largestSublists if there is no list which # contains list1 shouldAddList1 = True for list2 in largestSublists: if all(x in list1._list for x in list2._list): shouldAddList1 = False if (shouldAddList1): largestSublists.add(list1) matchedBaseLists = [] matchedBaseLists.extend(largestSublists) # for ( ListWrapper matchedDrawableList : matchedBaseLists) : # System.out.prln("After still: " + rectView.bound() + ", list: " # + matchedDrawableList.size() + ", " + matchedDrawableList) # notDrwableRectViews = [] for _list in matchedBaseLists: notDrwableRectViews = [ x for x in notDrwableRectViews if x not in _list._list ] minSize = RectUtil.minSizeListWrapper(matchedBaseLists) # Now take min size as the base then group them groups = [] for i in range(len(minSize)): listItem = ListItemMetadata() for matchedDrawableList in matchedBaseLists: listItem.baseViews.append(matchedDrawableList_list[i]) # listItem.baseViews.size > 0 listItem.bound = RectUtil.findBoundRectangle(listItem.baseViews) groups.append(listItem) # Find views overlap these bound of each list add put them o # the group processedView = [] for listItem in groups: for notDrawableView in notDrwableRectViews: if (not processedView.contains(notDrawableView) and RectUtil.ersectsNotInclude( listItem.bound, notDrawableView.bound())): listItem.additionalViews.add(notDrawableView) processedView.add(notDrawableView) for listItem in groups: # We got a new bound here, this time more aggressive => we # only include view belong to # to the new bound if len(listItem.additionalViews) > 0: listItem.bound = RectUtil.union( listItem.bound, RectUtil.findBoundRectangle(listItem.additionalViews)) for notDrawableView in notDrwableRectViews: if (not processedView.contains(notDrawableView) and RectUtil.contains(listItem.bound, notDrawableView.bound())): listItem.additionalViews.add(notDrawableView) processedView.add(notDrawableView) # check if this list is the expand version of the previous list # find the first list which have same size expandingList = None for i in len(self.mListViews): listItemMetadatas = self.mListViews[i].mListItemMetadatas if len(listItemMetadatas) == len(groups): includeMetada = True for j in range(len(groups)): if (not (groups[j].baseViews.containsAll( listItemMetadatas[j].baseViews) and groups[j].baseViews.size() > listItemMetadatas[j].baseViews.size())): includeMetada = False if (includeMetada): expandingList = self.mListViews[i] break if (expandingList != None): # System.out.prln("Expanding: " + rectView) self.mListViews[self.mListViews.index( expandingList)] = ListMetadataRoot(rectView, groups, alignmentType) else: validList = True for listItem in groups: allViews = [] allViews.extend(listItem.additionalViews) bound = RectUtil.findBoundRectangle(allViews) countListView = 0 for listMetadata in self.mListViews: # contain entire a list is okay but only one list item # is not okay if (RectUtil.ersects(listMetadata.bound(), bound) or RectUtil.contains(listMetadata.bound(), bound)): countListView += countListView if (countListView >= 1): validList = False break if (validList): if len(groups) > 0: self.mListViews.append( ListMetadataRoot(rectView, groups, alignmentType)) invalidLists = [] for listMetadataRoot in self.mListViews: if (not self.validateList(listMetadataRoot)): invalidLists.append(listMetadataRoot) self.mListViews = [x for x in self.mListViews if x not in invalidLists]