def createListLayoutCode(self, listMetadata, index, listView): baseListItemMetadata = listMetadata.mListItemMetadatas[0] # document = XmlUtil.createDocument() rootView = RectView(baseListItemMetadata.bound, None) for rectView in baseListItemMetadata.baseViews: rootView.addChild(rectView) for rectView in baseListItemMetadata.additionalViews: rootView.addChild(rectView) rootView.mChildren.sort( key=cmp_to_key(RectUtil.getTopBottomComparator())) rootView.mChildren.sort( key=cmp_to_key(RectUtil.getLeftRightComparator())) rootElement = XmlUtil.createRoot(self.mDipCalculator, FRAMELAYOUT_ELEMENT, rootView, self.mColorWriter) _map = {} self.addChildrenLayout(rootElement, rootView, listView.x, listView.y, _map) rectViews = RectUtil.toRects(rootView) rectViews.remove(rootView) XmlUtil.writeDocument( rootView, self.mOutProjectFolder + Constants.DEFAULT_LAYOUT_PATH + "/" + Constants.DEFAULT_LAYOUT_LIST_PREFIX + index + ".xml")
def accept(self, ocr): # Count to see how may child view this word have, if it only have # one child view. Test: # (1) This word is not messy. Mean that it did not # "intersect not include" with other views (except the child view) # (2) If the child view is too small compare with it foundChildRects = RectUtil.findChildRect(ocr.rect, self.mViews) if len(foundChildRects) == 1: rectView = foundChildRects[0] newList = [] if rectView in newList: newList.remove(rectView) findIntersectNotIncludeRect = RectUtil.findIntersectNotIncludeRect( ocr, newList) if len(findIntersectNotIncludeRect) == 1: iRect = findIntersectNotIncludeRect[0] if RectUtil.dimesionEqual( ocr, iRect, 0.2) and RectUtil.dimesionSmallerThan( rectView, ocr, 0.8): # this is wrong, ignore this word # DarkSalmon # http:#www.w3schools.com/tags/ref_color_tryit.asp?color=DarkSalmon tv = TextValidator.TextValidator( ocr, (233, 150, 122), False, "This word only have one child view, and the child view is too small compare with it" ) return tv return None
def pruneRemoveRedundantViewsInternal(self, parent, view): if (parent != None): # TODO: carefully test this logic # (1) If this view only have one children it the text view # And it size almost the same as the textview, so why we need # this container then, just replace it with the text view if len(view.mChildren) == 1: child = view.mChildren[0] if child.mType == RectViewTypes.VIEW_TYPE_TEXT and RectUtil.same( view, child, 0.5): return (view, child) elif child.mType == RectViewTypes.VIEW_TYPE_IMAGE and RectUtil.same( view, child, 0.5): return (view, child) # recursive to children # children = view.mChildren swapLists = [] for childView in view.mChildren: swapList = self.pruneRemoveRedundantViewsInternal(view, childView) swapLists.append(swapList) for pair in swapLists: view.mChildren = [x for x in view.mChildren if x is not pair[0]] if pair[1] is not None: view.mChildren.append(pair[1]) return (None, None)
def groupViewsForList(self, parentView, newChildren, viewTypeRect, minChidren): if len(newChildren) >= minChidren: bound = RectUtil.findBoundRectangle(newChildren) newParent = RectView(bound, None) newParent.mType = viewTypeRect # replace the parent at the location of the first child indexOf = parentView.mChildren.index(newChildren[0]) if indexOf > 0 and indexOf < len(parentView.mChildren): parentView.mChildren[indexOf] = newParent else: parentView.mChildren.append(newParent) # Now remove the rest parentView.mChildren = [ x for x in parentView.mChildren if x not in newChildren ] # Make sure there is no view is hidden under the new parent insideViews = RectUtil.contain(newParent, parentView.mChildren) parentView.mChildren = [ x for x in parentView.mChildren if x not in insideViews ] indexOfNewParent = parentView.mChildren.index(newParent) if (indexOfNewParent == len(parentView.mChildren) - 1): parentView.mChildren.extend(insideViews) else: parentView.mChildren.extend(indexOfNewParent + 1, insideViews) return newParent return None
def accept(self, ocr): # return None bound = ocr.bound() for view in self.mViews: # the box may has the word is too small compare with the word # itself. # If the word a children view which only have on child, we need # to verify if: # (1) This child view did not intersect with any other views if RectUtil.contains(bound, view.bound()) and len( view.mChildren) == 0: # make sure this view did not intersect with other view, # include is accepted in this case hasIntersection = False for otherView in self.mViews: if otherView != view and RectUtil.intersectsNotInclude( bound, otherView.bound()): hasIntersection = True break if not hasIntersection: if RectUtil.dimesionSmallerThan( view, ocr, 0.8): # this is wrong, ignore this word tv = TextValidator( ocr, CColor.Black, False, "The box may has the word is too small compare with the word itself" ) return tv return None
def removeChildren( src, view) : contours = [] children = view.mChildren # black out mask = np.ones(src.shape, np.int8) thisContours = [] if (view.contour != None) : # draw this contour contour = RectUtil.convertToParentCorrdinate(view, view.contour) thisContours.append(contour); # Imgproc.drawContours(mask, contours, -1, new Scalar(0), Core.FILLED); cv2.polylines(mask, thisContours, True, ColorUtil.getScalar(0),-1) # Extra and update x, y of children's contours and rects rects = [] for child in children: contour = child.contour if (contour == None) : rect = RectUtil.convertToParentCorrdinate(view,child) rects.append(rect) else : contour = RectUtil.convertToParentCorrdinateContour(view, contour) contours.append(contour) cv2.polylines(mask, contours, True, ColorUtil.getScalar(255),cv2.FILLED) # Fill children's inner contours with white and outer contours with # black # Imgproc.drawContours(mask, contours, -1, new Scalar(255), Core.FILLED); # Fill inner rects with white and outer rects with black for rect in rects: cv2.rectangle(mask, rect.tl(), rect.br(), ColorUtil.getScalar(255), cv2.FILLED) # inverse: black -> white and white -> black newMask = np.zeros(src.shape,np.int8) cv2.bitwise_not(mask, newMask) # let's create a new image now # crop = new (src.rows(), src.cols(), CvType.CV_8UC3); # set background to dominate color width = 0 height = 0 if len(src.shape) == 2 : height, width = src.shape else: height, width,channels = src.shape crop = np.zeros(src.shape, src.dtype) crop[:] = ColorUtil.getScalar(ColorUtil.findDominateColor(Rect(0, 0, width, height), src)) np.copyto(crop, src, 'unsafe', newMask.astype(bool)) return crop
def logListOverlay(self): isdebugMode = False if (isdebugMode): ImageUtil.log(RectUtil.toMapRects(self.mRootView), "pruneToCreateGroupTextWithList", self.mImage, True) ImageUtil.log(RectUtil.toMapRects(self.mRootView), self.mOutLogFolder, self.mFileName, "WithListOverlay", self.mImage, False)
def apply(self, info): if (not (info.mSecond.bound().tl_Pt().y - info.mFirst.bound().br_Pt().y <= self.mGroupDistanceVerticalTheshold)): return False for neighbour in info.mNeighbours: if (neighbour != info.mFirst and neighbour != info.mSecond and RectUtil.above(info.mSecond, neighbour) and RectUtil.below(info.mFirst, neighbour)): return False return True
def getSmallestBoundRect(mRectBoundOfTexts, rect): smalledRect = Rect() # check if there is any rectangle contains with this rect for erRect in mRectBoundOfTexts: if (RectUtil.contains(erRect, rect)): if (smalledRect.area() == 0): smalledRect = erRect elif (RectUtil.contains(smalledRect, erRect)): smalledRect = erRect return smalledRect
def apply(self, info): if (not (info.mSecond.bound().tl_Pt().x - info.mFirst.bound().br_Pt().x <= self.mGroupDistanceVerticalTheshold)): return False for neighbour in info.mNeighbours(): if (neighbour != info.mFirst and neighbour != info.mSecond and RectUtil.left(info.mSecond.bound(), neighbour.bound()) and RectUtil.right(info.mFirst.bound(), neighbour.bound())): return False return True
def __init__(self, image, img_gray, biMapViewRect, ocr, dipCalculator): self.mRgbaImage = image self.grayImage = img_gray self.mBiMapViewRect = biMapViewRect self.mOcr = ocr self.mDipCalculator = dipCalculator values = set([biMapViewRect[k] for k in biMapViewRect]) self.mViewBounds = [] self.mViewBounds.extend(values) RectUtil.sortLeftRightTopBottom(self.mViewBounds) self.mViews = [k for k in biMapViewRect] self.isDebugMode = True
def removeChildrenAndCreateTransparentBackground( src, view) : contours = [] children = view.mChildren # black out mask = np.ones(src.shape,np.int8) alpha = np.zeros(src.shape,np.int8) if (view.contour != None) : # draw this contour thisContours = [] contour = RectUtil.convertToParentCorrdinate(view, view.contour) thisContours.append(contour) cv2.polylines(mask, thisContours, True, ColorUtil.getScalar(0),cv2.FILLED) # Imgproc.drawContours(mask, contours, -1, new Scalar(0), Core.FILLED); # Extra and update x, y of children's contours and rects rects = [] for child in children: contour = child.contour if (contour == None) : bound = RectUtil.convertToParentCorrdinate(view, child) (x1,y1,width,height)= cv2.boundingRect(bound) rect = Rect(x1,y1,width,height) rects.append(rect) else : contour = RectUtil.convertToParentCorrdinate(view, contour); contours.append(contour) # Fill children's inner contours with white and outer contours with # black # Imgproc.drawContours(mask, contours, -1, new Scalar(255), Core.FILLED); cv2.polylines(mask, contours, True, ColorUtil.getScalar(255),cv2.FILLED) # Fill inner rects with white and outer rects with black for rect in rects: cv2.rectangle(mask, rect.tl(), rect.br(), ColorUtil.getScalar(255), cv2.FILLED) # inverse: black -> white and white -> black newMask = np.zeros(src.shape, np.int8) cv2.bitwise_not(mask, newMask) retval, alpha = cv2.threshold(newMask,100,255,cv2.THRESH_BINARY) b,g,r = cv2.split(src) merge = cv2.merge((b,g,r,alpha)) return merge
def createHierachy(rects, width, height): rootObj = RectObj(Rect(0, 0, width, height)) sortedRectObjs = sorted(rects, key=lambda x: x.rectArea) elementLength = len(sortedRectObjs) if (elementLength == 1): rootObj.mChildren.append(sortedRectObjs[0]) return rootObj for i in range(elementLength - 1): item = sortedRectObjs[i] validElement = True isChild = False # print(item.rectArea) for j in range(i + 1, elementLength): parItem = sortedRectObjs[j] if parItem != item: item, validElement, isChild = RectUtil.fixHierarchy( parItem, item, width, height) if isChild: parItem.mChildren.append(item) break if not validElement: break if validElement and not isChild: rootObj.mChildren.append(item) rootObj.mChildren.append(sortedRectObjs[elementLength - 1]) searchForTextButton(rootObj) return rootObj
def isOverlapText(self, rect, confident): for ocrTextWrapper in self.mOcrTextWrappers: bound = ocrTextWrapper.bound() if (ocrTextWrapper.getConfidence() >= confident and RectUtil.intersects(rect, bound)): return True return False
def addTextToHierarchyInternal(self, view, blocks): children = view.mChildren for childView in children: self.addTextToHierarchyInternal(childView, blocks) childBlocks = TextProcessorUtil.getTextAndRemove(view, blocks) if (len(childBlocks) == 0): return removedChildren = [] for rectView in children: removed = False for ocrBlock in childBlocks: # // if it overlap a view but not include in it, we should just # // remove these views # // if children already has text, we should keep them # // *** There is situation in which the text right at the edge so # // the vision box # // need to expand just a bit (1 px each dimension) to cover the # // text if (not RectUtil.contains(RectUtil.expand1Px(rectView.bound()), ocrBlock.bound()) and RectUtil.intersects(ocrBlock.bound(), rectView.bound()) and not rectView.hasTextRecusive()): removed = True break if (removed): removedChildren.append(rectView) view.mChildren = [ x for x in view.mChildren if x not in removedChildren ] view.mTextWithLocations = childBlocks view.mTextChildren = removedChildren # // We want to remove all grand children vision box too. # // This is the error we found in TimeHop iOS app. leafNodes = RectUtil.getLeafNodes(view) for rectViewPair in leafNodes: for text in childBlocks: if (RectUtil.contains(text, rectViewPair[0].rect, 0.75) and rectViewPair[0].mChildren is not None): rectViewPair[0].mChildren = rectViewPair[ 0].mChildren.remove(rectViewPair[1]) break
def getBlockWithLocation(self, rect): wrappers = [] for ocrTextWrapper in self.mOcrBlockWrappers: bound = ocrTextWrapper.rect if (RectUtil.contains(rect, bound)): wrappers.append(OCRTextWrapper.OCRTextWrapper(ocrTextWrapper)) return wrappers
def getWordsIn(self, rect): wrappers = [] for ocrTextWrapper in self.mOcrTextWrappers: bound = ocrTextWrapper.bound() if (RectUtil.contains(rect, bound)): wrappers.append(OCRTextWrapper.OCRTextWrapper(ocrTextWrapper)) return wrappers
def doFilderInternal(self,root, anotateMap): # @SuppressWarnings("unchecked") elements = [elem for elem in root.iter()] for node in elements: self.doFilderInternal(Element(node), anotateMap) if (self.isDefaultElement(root)): datas = [] for node in elements: child = Element(node) datas.append(anotateMap[child]) if len(elements) <= 1 : return root # // Check if children are overlapped if (not RectUtil.noOvelap(datas)): return root # System.out.println("noOvelap"); # // check if children order horizontally # if (RectUtil.isOrderHorizontally(datas)) { # # # } if (RectUtil.isOrderVertically(datas)): # check if children order vertically root.setName(LayoutCreator.RELATIVELAYOUT_ELEMENT); # // First sort all children vertically datas.sort(key=cmp_to_key( RectUtil.getLeftRightComparator())) # for (ElementData elementData : datas) { # # } else: return root; # for (Node node : elements) { # Element child = (Element) node; # # } # } return root;
def logListOverlay(self): mapRects = RectUtil.toMapRects(self.mRootView) for metadataRoot in self.mListViews: for listItemMetadata in metadataRoot.getListItemMetadatas(): mapRects.update(RectUtil.toMapRects( listItemMetadata.baseViews)) mapRects.update( RectUtil.toMapRects(listItemMetadata.additionalViews)) # Add List item color color = RectUtil.getColorWrapperBaseOnType( RectView.VIEW_TYPE_LIST_ITEM) iRects = mapRects[color] if (iRects == None): iRects = [] mapRects[color] = iRects iRects.append(RectUtil.toIRect(listItemMetadata.bound)) mapRects[color] = iRects
def getTextAndRemove(viewBound, blocks): childTexts = [] for ocrTextWrapper in blocks: bound = ocrTextWrapper.bound() if (RectUtil.contains(viewBound, bound)): childTexts.append(ocrTextWrapper) blocks = [x for x in blocks if x not in childTexts] return childTexts
def save(self): for key in self.drawableInfos: contour = self.drawableInfos[key].rectView.contour if (contour != None and not RectView.isContanerView(self.drawableInfos[key].rectView)): contour = RectUtil.convertToParentCorrdinate(self.drawableInfos[key].rectView, contour) overlay = ImageUtil.createTransparentBackground(self.drawableInfos[key].mat, contour) cv2.writePng(self.drawableInfos[key].path, overlay) else: cv2.imwrite(self.drawableInfos[key].path, self.drawableInfos[key].mat)
def isValidList(self, viewList): if len(viewList) < Constants.LAYOUT_MIN_ACCEPTABLE_LIST_SIZE: return False bound = RectUtil.findBoundRectangle(viewList) heightDip = self.mDipCalculator.pxToHeightDip(bound.height) widthDip = self.mDipCalculator.pxToWidthDip(bound.width) area = heightDip * widthDip return area >= Constants.MIN_SINGLE_LIST_AREA
def isTextViewOrTextViewContainer(self, rectView): if (rectView.mType == RectViewTypes.VIEW_TYPE_TEXT): return True if len( rectView.mChildren ) == 1 and rectView.mChildren[0].mType == RectViewTypes.VIEW_TYPE_TEXT: return RectUtil.same(rectView, rectView.mChildren[0], 0.1) return False
def hasParent(ocrTextWrapper, ocrWrappers): # keep word did not have children hasParent = False for otherOcrTextWrapper in ocrWrappers: if otherOcrTextWrapper != ocrTextWrapper and RectUtil.contains( otherOcrTextWrapper.rect, ocrTextWrapper.rect): return True return hasParent
def accept(self,ocr): bound = ocr.bound() # return None for view in self.mViews: # woa this word is big and have a lot of children, not good # this may okay with url or special texts if RectUtil.contains(bound, view.bound()) and len(view.mChildren) > 0: tv = TextValidator(ocr, CColor.Cyan, False, "This word is big and have a lot of children" + str(len(view.mChildren))) return tv return None
def initLine(self): self.mOcrLineWrappers = self.baseInit(RIL.TEXTLINE) invalidLineWrappers = [] # a line cannot contain another lines for ocrLine in self.mOcrLineWrappers: for otherOcrLine in self.mOcrLineWrappers: if (ocrLine != otherOcrLine and RectUtil.contains( ocrLine.bound(), otherOcrLine.bound())): invalidLineWrappers.append(ocrLine) self.mOcrLineWrappers = [ x for x in self.mOcrLineWrappers if x not in invalidLineWrappers ]
def reorganizeOrder(self, view): children = view.getChildren() orderViewWrapers = {} for childView in children: orderViewWraper = OrderViewWraper(childView, children.index(childView)) orderViewWrapers[childView] = orderViewWraper for childView in children: for otherChildView in children: # we want compare reference here! # check if we need to move the rectangle to lower level if (otherChildView != childView): childBound = childView.bound() otherChildBound = otherChildView.bound() if (RectUtil.intersects(childBound, otherChildBound)): otherGrandChildren = otherChildView.getChildren() # this child did not overlap any children of the # other child overlap overlap = False for rawView in otherGrandChildren: if (RectUtil.intersects(childBound, rawView.bound())): overlap = True break if (not overlap and children.index(childView) < children.index(otherChildView)): orderViewWraper = orderViewWrapers[childView] otherOrderViewWraper = orderViewWrapers[ otherChildView] orderViewWraper.otherView = otherOrderViewWraper sortedOrderViewWrapers = orderViewWrapers.values() sortedOrderViewWrapers.sort(self.OrderViewWrapers) children = [] for orderViewWraper in sortedOrderViewWrapers: children.append(orderViewWraper.view)
def reCalculateBoundBaseOnWordList(self): if (len(self.words) == 0): return None firstWord = self.words[0] unionRect = firstWord.rect for i in range(1, len(self.words)): word = self.words[i] unionRect = RectUtil.union(unionRect, word.rect) self.x = unionRect.x self.y = unionRect.y self.width = unionRect.width self.height = unionRect.height return unionRect
def accept(self, ocr): rects = [] ocrBound = ocr.bound() for view in self.mViews: # accept overlap here if not self.mDipCalculator.isViewToBeIgnoreView( view) and RectUtil.contains(ocrBound, view.bound(), 0.5): rects.append(view.bound()) if len(rects) >= 2: RectUtil.sortTopBottom(rects) # distance between 2 characters are greater than the character # itself this have to be true for all of character invalidWordWithChars = False for i in range(len(rects) - 1): current = rects[i] next_rect = rects[i + 1] # ignore intersect one if RectUtil.intersects(current, next_rect): continue if current.width < next_rect.x - (current.x + current.width): invalidWordWithChars = True else: invalidWordWithChars = False break if invalidWordWithChars: # SkyBlue tv = TextValidator.TextValidator( ocr, (135, 206, 235), False, "distance between 2 characters are greater than the character itself: " ) return tv return None
def validateList(self, listMetadataRoot): alignmentType = listMetadataRoot.mAlignmentType groups = listMetadataRoot.mListItemMetadatas # We only support more than one base view for listItem in groups: if len(listItem.baseViews) <= 1: return False # and all base views of each list item should have same size currentSize = -sys.maxint - 1 for listItem in groups: if (currentSize == -sys.maxint - 1): currentSize = len(listItem.baseViews) elif (currentSize != listItem.baseViews.size()): return False # They have to be align if alignmentType == RectUtil.ALIGNMENT_RIGHT: for i in range(len(groups) - 1): current = groups[i] _next = groups[i + 1] for currentBase in current.baseViews: for nextBase in _next.baseViews: if (not RectUtil.above(currentBase, nextBase)): return False if alignmentType == RectUtil.ALIGNMENT_BOTTOM: for i in range(len(groups) - 1): current = groups[i] _next = groups[i + 1] for currentBase in current.baseViews: for nextBase in _next.baseViews: if (not RectUtil.onTheLeft(currentBase, nextBase)): return False return True