def createListLayoutCode(self, listMetadata, index, listView):
baseListItemMetadata = listMetadata.mListItemMetadatas[0]
# document = XmlUtil.createDocument()
rootView = RectView(baseListItemMetadata.bound, None)
for rectView in baseListItemMetadata.baseViews:
rootView.addChild(rectView)
for rectView in baseListItemMetadata.additionalViews:
rootView.addChild(rectView)
rootView.mChildren.sort(
key=cmp_to_key(RectUtil.getTopBottomComparator()))
rootView.mChildren.sort(
key=cmp_to_key(RectUtil.getLeftRightComparator()))
rootElement = XmlUtil.createRoot(self.mDipCalculator,
FRAMELAYOUT_ELEMENT, rootView,
self.mColorWriter)
_map = {}
self.addChildrenLayout(rootElement, rootView, listView.x, listView.y,
_map)
rectViews = RectUtil.toRects(rootView)
rectViews.remove(rootView)
XmlUtil.writeDocument(
rootView, self.mOutProjectFolder + Constants.DEFAULT_LAYOUT_PATH +
"/" + Constants.DEFAULT_LAYOUT_LIST_PREFIX + index + ".xml")
def accept(self, ocr):
# Count to see how may child view this word have, if it only have
# one child view. Test:
# (1) This word is not messy. Mean that it did not
# "intersect not include" with other views (except the child view)
# (2) If the child view is too small compare with it
foundChildRects = RectUtil.findChildRect(ocr.rect, self.mViews)
if len(foundChildRects) == 1:
rectView = foundChildRects[0]
newList = []
if rectView in newList:
newList.remove(rectView)
findIntersectNotIncludeRect = RectUtil.findIntersectNotIncludeRect(
ocr, newList)
if len(findIntersectNotIncludeRect) == 1:
iRect = findIntersectNotIncludeRect[0]
if RectUtil.dimesionEqual(
ocr, iRect, 0.2) and RectUtil.dimesionSmallerThan(
rectView, ocr, 0.8):
# this is wrong, ignore this word
# DarkSalmon
# http:#www.w3schools.com/tags/ref_color_tryit.asp?color=DarkSalmon
tv = TextValidator.TextValidator(
ocr, (233, 150, 122), False,
"This word only have one child view, and the child view is too small compare with it"
)
return tv
return None
def pruneRemoveRedundantViewsInternal(self, parent, view):
if (parent != None):
# TODO: carefully test this logic
# (1) If this view only have one children it the text view
# And it size almost the same as the textview, so why we need
# this container then, just replace it with the text view
if len(view.mChildren) == 1:
child = view.mChildren[0]
if child.mType == RectViewTypes.VIEW_TYPE_TEXT and RectUtil.same(
view, child, 0.5):
return (view, child)
elif child.mType == RectViewTypes.VIEW_TYPE_IMAGE and RectUtil.same(
view, child, 0.5):
return (view, child)
# recursive to children
# children = view.mChildren
swapLists = []
for childView in view.mChildren:
swapList = self.pruneRemoveRedundantViewsInternal(view, childView)
swapLists.append(swapList)
for pair in swapLists:
view.mChildren = [x for x in view.mChildren if x is not pair[0]]
if pair[1] is not None:
view.mChildren.append(pair[1])
return (None, None)
def groupViewsForList(self, parentView, newChildren, viewTypeRect,
minChidren):
if len(newChildren) >= minChidren:
bound = RectUtil.findBoundRectangle(newChildren)
newParent = RectView(bound, None)
newParent.mType = viewTypeRect
# replace the parent at the location of the first child
indexOf = parentView.mChildren.index(newChildren[0])
if indexOf > 0 and indexOf < len(parentView.mChildren):
parentView.mChildren[indexOf] = newParent
else:
parentView.mChildren.append(newParent)
# Now remove the rest
parentView.mChildren = [
x for x in parentView.mChildren if x not in newChildren
]
# Make sure there is no view is hidden under the new parent
insideViews = RectUtil.contain(newParent, parentView.mChildren)
parentView.mChildren = [
x for x in parentView.mChildren if x not in insideViews
]
indexOfNewParent = parentView.mChildren.index(newParent)
if (indexOfNewParent == len(parentView.mChildren) - 1):
parentView.mChildren.extend(insideViews)
else:
parentView.mChildren.extend(indexOfNewParent + 1, insideViews)
return newParent
return None
def accept(self, ocr):
# return None
bound = ocr.bound()
for view in self.mViews:
# the box may has the word is too small compare with the word
# itself.
# If the word a children view which only have on child, we need
# to verify if:
# (1) This child view did not intersect with any other views
if RectUtil.contains(bound, view.bound()) and len(
view.mChildren) == 0:
# make sure this view did not intersect with other view,
# include is accepted in this case
hasIntersection = False
for otherView in self.mViews:
if otherView != view and RectUtil.intersectsNotInclude(
bound, otherView.bound()):
hasIntersection = True
break
if not hasIntersection:
if RectUtil.dimesionSmallerThan(
view, ocr, 0.8): # this is wrong, ignore this word
tv = TextValidator(
ocr, CColor.Black, False,
"The box may has the word is too small compare with the word itself"
)
return tv
return None
def removeChildren( src, view) :
contours = []
children = view.mChildren
# black out
mask = np.ones(src.shape, np.int8)
thisContours = []
if (view.contour != None) :
# draw this contour
contour = RectUtil.convertToParentCorrdinate(view, view.contour)
thisContours.append(contour);
# Imgproc.drawContours(mask, contours, -1, new Scalar(0), Core.FILLED);
cv2.polylines(mask, thisContours, True, ColorUtil.getScalar(0),-1)
# Extra and update x, y of children's contours and rects
rects = []
for child in children:
contour = child.contour
if (contour == None) :
rect = RectUtil.convertToParentCorrdinate(view,child)
rects.append(rect)
else :
contour = RectUtil.convertToParentCorrdinateContour(view, contour)
contours.append(contour)
cv2.polylines(mask, contours, True, ColorUtil.getScalar(255),cv2.FILLED)
# Fill children's inner contours with white and outer contours with
# black
# Imgproc.drawContours(mask, contours, -1, new Scalar(255), Core.FILLED);
# Fill inner rects with white and outer rects with black
for rect in rects:
cv2.rectangle(mask, rect.tl(), rect.br(), ColorUtil.getScalar(255), cv2.FILLED)
# inverse: black -> white and white -> black
newMask = np.zeros(src.shape,np.int8)
cv2.bitwise_not(mask, newMask)
# let's create a new image now
# crop = new (src.rows(), src.cols(), CvType.CV_8UC3);
# set background to dominate color
width = 0
height = 0
if len(src.shape) == 2 :
height, width = src.shape
else:
height, width,channels = src.shape
crop = np.zeros(src.shape, src.dtype)
crop[:] = ColorUtil.getScalar(ColorUtil.findDominateColor(Rect(0, 0, width, height), src))
np.copyto(crop, src, 'unsafe', newMask.astype(bool))
return crop
def logListOverlay(self):
isdebugMode = False
if (isdebugMode):
ImageUtil.log(RectUtil.toMapRects(self.mRootView),
"pruneToCreateGroupTextWithList", self.mImage, True)
ImageUtil.log(RectUtil.toMapRects(self.mRootView),
self.mOutLogFolder, self.mFileName,
"WithListOverlay", self.mImage, False)
def apply(self, info):
if (not (info.mSecond.bound().tl_Pt().y - info.mFirst.bound().br_Pt().y
<= self.mGroupDistanceVerticalTheshold)):
return False
for neighbour in info.mNeighbours:
if (neighbour != info.mFirst and neighbour != info.mSecond
and RectUtil.above(info.mSecond, neighbour)
and RectUtil.below(info.mFirst, neighbour)):
return False
return True
def getSmallestBoundRect(mRectBoundOfTexts, rect):
smalledRect = Rect()
# check if there is any rectangle contains with this rect
for erRect in mRectBoundOfTexts:
if (RectUtil.contains(erRect, rect)):
if (smalledRect.area() == 0):
smalledRect = erRect
elif (RectUtil.contains(smalledRect, erRect)):
smalledRect = erRect
return smalledRect
def apply(self, info):
if (not (info.mSecond.bound().tl_Pt().x - info.mFirst.bound().br_Pt().x
<= self.mGroupDistanceVerticalTheshold)):
return False
for neighbour in info.mNeighbours():
if (neighbour != info.mFirst and neighbour != info.mSecond
and RectUtil.left(info.mSecond.bound(), neighbour.bound())
and RectUtil.right(info.mFirst.bound(),
neighbour.bound())):
return False
return True
def __init__(self, image, img_gray, biMapViewRect, ocr, dipCalculator):
self.mRgbaImage = image
self.grayImage = img_gray
self.mBiMapViewRect = biMapViewRect
self.mOcr = ocr
self.mDipCalculator = dipCalculator
values = set([biMapViewRect[k] for k in biMapViewRect])
self.mViewBounds = []
self.mViewBounds.extend(values)
RectUtil.sortLeftRightTopBottom(self.mViewBounds)
self.mViews = [k for k in biMapViewRect]
self.isDebugMode = True
def removeChildrenAndCreateTransparentBackground( src, view) :
contours = []
children = view.mChildren
# black out
mask = np.ones(src.shape,np.int8)
alpha = np.zeros(src.shape,np.int8)
if (view.contour != None) :
# draw this contour
thisContours = []
contour = RectUtil.convertToParentCorrdinate(view, view.contour)
thisContours.append(contour)
cv2.polylines(mask, thisContours, True, ColorUtil.getScalar(0),cv2.FILLED)
# Imgproc.drawContours(mask, contours, -1, new Scalar(0), Core.FILLED);
# Extra and update x, y of children's contours and rects
rects = []
for child in children:
contour = child.contour
if (contour == None) :
bound = RectUtil.convertToParentCorrdinate(view, child)
(x1,y1,width,height)= cv2.boundingRect(bound)
rect = Rect(x1,y1,width,height)
rects.append(rect)
else :
contour = RectUtil.convertToParentCorrdinate(view, contour);
contours.append(contour)
# Fill children's inner contours with white and outer contours with
# black
# Imgproc.drawContours(mask, contours, -1, new Scalar(255), Core.FILLED);
cv2.polylines(mask, contours, True, ColorUtil.getScalar(255),cv2.FILLED)
# Fill inner rects with white and outer rects with black
for rect in rects:
cv2.rectangle(mask, rect.tl(), rect.br(), ColorUtil.getScalar(255), cv2.FILLED)
# inverse: black -> white and white -> black
newMask = np.zeros(src.shape, np.int8)
cv2.bitwise_not(mask, newMask)
retval, alpha = cv2.threshold(newMask,100,255,cv2.THRESH_BINARY)
b,g,r = cv2.split(src)
merge = cv2.merge((b,g,r,alpha))
return merge
def createHierachy(rects, width, height):
rootObj = RectObj(Rect(0, 0, width, height))
sortedRectObjs = sorted(rects, key=lambda x: x.rectArea)
elementLength = len(sortedRectObjs)
if (elementLength == 1):
rootObj.mChildren.append(sortedRectObjs[0])
return rootObj
for i in range(elementLength - 1):
item = sortedRectObjs[i]
validElement = True
isChild = False
# print(item.rectArea)
for j in range(i + 1, elementLength):
parItem = sortedRectObjs[j]
if parItem != item:
item, validElement, isChild = RectUtil.fixHierarchy(
parItem, item, width, height)
if isChild:
parItem.mChildren.append(item)
break
if not validElement:
break
if validElement and not isChild:
rootObj.mChildren.append(item)
rootObj.mChildren.append(sortedRectObjs[elementLength - 1])
searchForTextButton(rootObj)
return rootObj
def isOverlapText(self, rect, confident):
for ocrTextWrapper in self.mOcrTextWrappers:
bound = ocrTextWrapper.bound()
if (ocrTextWrapper.getConfidence() >= confident
and RectUtil.intersects(rect, bound)):
return True
return False
def addTextToHierarchyInternal(self, view, blocks):
children = view.mChildren
for childView in children:
self.addTextToHierarchyInternal(childView, blocks)
childBlocks = TextProcessorUtil.getTextAndRemove(view, blocks)
if (len(childBlocks) == 0):
return
removedChildren = []
for rectView in children:
removed = False
for ocrBlock in childBlocks:
# // if it overlap a view but not include in it, we should just
# // remove these views
# // if children already has text, we should keep them
# // *** There is situation in which the text right at the edge so
# // the vision box
# // need to expand just a bit (1 px each dimension) to cover the
# // text
if (not RectUtil.contains(RectUtil.expand1Px(rectView.bound()),
ocrBlock.bound())
and RectUtil.intersects(ocrBlock.bound(),
rectView.bound())
and not rectView.hasTextRecusive()):
removed = True
break
if (removed):
removedChildren.append(rectView)
view.mChildren = [
x for x in view.mChildren if x not in removedChildren
]
view.mTextWithLocations = childBlocks
view.mTextChildren = removedChildren
# // We want to remove all grand children vision box too.
# // This is the error we found in TimeHop iOS app.
leafNodes = RectUtil.getLeafNodes(view)
for rectViewPair in leafNodes:
for text in childBlocks:
if (RectUtil.contains(text, rectViewPair[0].rect, 0.75)
and rectViewPair[0].mChildren is not None):
rectViewPair[0].mChildren = rectViewPair[
0].mChildren.remove(rectViewPair[1])
break
def getBlockWithLocation(self, rect):
wrappers = []
for ocrTextWrapper in self.mOcrBlockWrappers:
bound = ocrTextWrapper.rect
if (RectUtil.contains(rect, bound)):
wrappers.append(OCRTextWrapper.OCRTextWrapper(ocrTextWrapper))
return wrappers
def getWordsIn(self, rect):
wrappers = []
for ocrTextWrapper in self.mOcrTextWrappers:
bound = ocrTextWrapper.bound()
if (RectUtil.contains(rect, bound)):
wrappers.append(OCRTextWrapper.OCRTextWrapper(ocrTextWrapper))
return wrappers
def doFilderInternal(self,root, anotateMap):
# @SuppressWarnings("unchecked")
elements = [elem for elem in root.iter()]
for node in elements:
self.doFilderInternal(Element(node), anotateMap)
if (self.isDefaultElement(root)):
datas = []
for node in elements:
child = Element(node)
datas.append(anotateMap[child])
if len(elements) <= 1 :
return root
# // Check if children are overlapped
if (not RectUtil.noOvelap(datas)):
return root
# System.out.println("noOvelap");
# // check if children order horizontally
# if (RectUtil.isOrderHorizontally(datas)) {
#
#
# }
if (RectUtil.isOrderVertically(datas)):
# check if children order vertically
root.setName(LayoutCreator.RELATIVELAYOUT_ELEMENT);
# // First sort all children vertically
datas.sort(key=cmp_to_key( RectUtil.getLeftRightComparator()))
# for (ElementData elementData : datas) {
#
# }
else:
return root;
# for (Node node : elements) {
# Element child = (Element) node;
#
# }
# }
return root;
def logListOverlay(self):
mapRects = RectUtil.toMapRects(self.mRootView)
for metadataRoot in self.mListViews:
for listItemMetadata in metadataRoot.getListItemMetadatas():
mapRects.update(RectUtil.toMapRects(
listItemMetadata.baseViews))
mapRects.update(
RectUtil.toMapRects(listItemMetadata.additionalViews))
# Add List item color
color = RectUtil.getColorWrapperBaseOnType(
RectView.VIEW_TYPE_LIST_ITEM)
iRects = mapRects[color]
if (iRects == None):
iRects = []
mapRects[color] = iRects
iRects.append(RectUtil.toIRect(listItemMetadata.bound))
mapRects[color] = iRects
def getTextAndRemove(viewBound, blocks):
childTexts = []
for ocrTextWrapper in blocks:
bound = ocrTextWrapper.bound()
if (RectUtil.contains(viewBound, bound)):
childTexts.append(ocrTextWrapper)
blocks = [x for x in blocks if x not in childTexts]
return childTexts
def save(self):
for key in self.drawableInfos:
contour = self.drawableInfos[key].rectView.contour
if (contour != None and not RectView.isContanerView(self.drawableInfos[key].rectView)):
contour = RectUtil.convertToParentCorrdinate(self.drawableInfos[key].rectView, contour)
overlay = ImageUtil.createTransparentBackground(self.drawableInfos[key].mat, contour)
cv2.writePng(self.drawableInfos[key].path, overlay)
else:
cv2.imwrite(self.drawableInfos[key].path, self.drawableInfos[key].mat)
def isValidList(self, viewList):
if len(viewList) < Constants.LAYOUT_MIN_ACCEPTABLE_LIST_SIZE:
return False
bound = RectUtil.findBoundRectangle(viewList)
heightDip = self.mDipCalculator.pxToHeightDip(bound.height)
widthDip = self.mDipCalculator.pxToWidthDip(bound.width)
area = heightDip * widthDip
return area >= Constants.MIN_SINGLE_LIST_AREA
def isTextViewOrTextViewContainer(self, rectView):
if (rectView.mType == RectViewTypes.VIEW_TYPE_TEXT):
return True
if len(
rectView.mChildren
) == 1 and rectView.mChildren[0].mType == RectViewTypes.VIEW_TYPE_TEXT:
return RectUtil.same(rectView, rectView.mChildren[0], 0.1)
return False
def hasParent(ocrTextWrapper, ocrWrappers):
# keep word did not have children
hasParent = False
for otherOcrTextWrapper in ocrWrappers:
if otherOcrTextWrapper != ocrTextWrapper and RectUtil.contains(
otherOcrTextWrapper.rect, ocrTextWrapper.rect):
return True
return hasParent
def accept(self,ocr):
bound = ocr.bound()
# return None
for view in self.mViews:
# woa this word is big and have a lot of children, not good
# this may okay with url or special texts
if RectUtil.contains(bound, view.bound()) and len(view.mChildren) > 0:
tv = TextValidator(ocr, CColor.Cyan, False, "This word is big and have a lot of children" + str(len(view.mChildren)))
return tv
return None
def initLine(self):
self.mOcrLineWrappers = self.baseInit(RIL.TEXTLINE)
invalidLineWrappers = []
# a line cannot contain another lines
for ocrLine in self.mOcrLineWrappers:
for otherOcrLine in self.mOcrLineWrappers:
if (ocrLine != otherOcrLine and RectUtil.contains(
ocrLine.bound(), otherOcrLine.bound())):
invalidLineWrappers.append(ocrLine)
self.mOcrLineWrappers = [
x for x in self.mOcrLineWrappers if x not in invalidLineWrappers
]
def reorganizeOrder(self, view):
children = view.getChildren()
orderViewWrapers = {}
for childView in children:
orderViewWraper = OrderViewWraper(childView,
children.index(childView))
orderViewWrapers[childView] = orderViewWraper
for childView in children:
for otherChildView in children:
# we want compare reference here!
# check if we need to move the rectangle to lower level
if (otherChildView != childView):
childBound = childView.bound()
otherChildBound = otherChildView.bound()
if (RectUtil.intersects(childBound, otherChildBound)):
otherGrandChildren = otherChildView.getChildren()
# this child did not overlap any children of the
# other child overlap
overlap = False
for rawView in otherGrandChildren:
if (RectUtil.intersects(childBound,
rawView.bound())):
overlap = True
break
if (not overlap and children.index(childView) <
children.index(otherChildView)):
orderViewWraper = orderViewWrapers[childView]
otherOrderViewWraper = orderViewWrapers[
otherChildView]
orderViewWraper.otherView = otherOrderViewWraper
sortedOrderViewWrapers = orderViewWrapers.values()
sortedOrderViewWrapers.sort(self.OrderViewWrapers)
children = []
for orderViewWraper in sortedOrderViewWrapers:
children.append(orderViewWraper.view)
def reCalculateBoundBaseOnWordList(self):
if (len(self.words) == 0):
return None
firstWord = self.words[0]
unionRect = firstWord.rect
for i in range(1, len(self.words)):
word = self.words[i]
unionRect = RectUtil.union(unionRect, word.rect)
self.x = unionRect.x
self.y = unionRect.y
self.width = unionRect.width
self.height = unionRect.height
return unionRect
def accept(self, ocr):
rects = []
ocrBound = ocr.bound()
for view in self.mViews:
# accept overlap here
if not self.mDipCalculator.isViewToBeIgnoreView(
view) and RectUtil.contains(ocrBound, view.bound(), 0.5):
rects.append(view.bound())
if len(rects) >= 2:
RectUtil.sortTopBottom(rects)
# distance between 2 characters are greater than the character
# itself this have to be true for all of character
invalidWordWithChars = False
for i in range(len(rects) - 1):
current = rects[i]
next_rect = rects[i + 1]
# ignore intersect one
if RectUtil.intersects(current, next_rect):
continue
if current.width < next_rect.x - (current.x + current.width):
invalidWordWithChars = True
else:
invalidWordWithChars = False
break
if invalidWordWithChars:
# SkyBlue
tv = TextValidator.TextValidator(
ocr, (135, 206, 235), False,
"distance between 2 characters are greater than the character itself: "
)
return tv
return None
def validateList(self, listMetadataRoot):
alignmentType = listMetadataRoot.mAlignmentType
groups = listMetadataRoot.mListItemMetadatas
# We only support more than one base view
for listItem in groups:
if len(listItem.baseViews) <= 1:
return False
# and all base views of each list item should have same size
currentSize = -sys.maxint - 1
for listItem in groups:
if (currentSize == -sys.maxint - 1):
currentSize = len(listItem.baseViews)
elif (currentSize != listItem.baseViews.size()):
return False
# They have to be align
if alignmentType == RectUtil.ALIGNMENT_RIGHT:
for i in range(len(groups) - 1):
current = groups[i]
_next = groups[i + 1]
for currentBase in current.baseViews:
for nextBase in _next.baseViews:
if (not RectUtil.above(currentBase, nextBase)):
return False
if alignmentType == RectUtil.ALIGNMENT_BOTTOM:
for i in range(len(groups) - 1):
current = groups[i]
_next = groups[i + 1]
for currentBase in current.baseViews:
for nextBase in _next.baseViews:
if (not RectUtil.onTheLeft(currentBase, nextBase)):
return False
return True
