def run(self, doc):
"""
take a set of line in a page and mine it
"""
self.doc = doc
# use the lite version
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
if self.bManual:
self.processWithTemplate(self.manualPattern, self.lPages)
else:
# self.mainMining(self.lPages)
for page in self.lPages:
print("page")
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
self.columnMining(table)
self.addTagProcessToMetadata(self.doc)
return self.doc
def run(self, domDoc):
"""
conversion
"""
ODoc = XMLDSDocument()
# ODoc.lastPage=1
ODoc.loadFromDom(domDoc)
lPageXmlDoc = []
lPages = ODoc.getPages()
for page in lPages:
# print("%s %s"%(page, page.getAttribute('imageFilename')))
try:
filename = os.path.basename(page.getAttribute('imageFilename'))
except:
filename = "fakename"
pageXmlDoc, pageNode = PageXml.createPageXmlDocument(
creatorName='NLE',
filename=filename,
imgW=convertDot2Pixel(self.dpi, page.getWidth()),
imgH=convertDot2Pixel(self.dpi, page.getHeight()))
self.pageXmlNS = etree.QName(pageXmlDoc.getroot()).namespace
if self.bRegionOnly:
self.convertOnlyRegion(page, pageNode)
else:
self.convertDSPage(page, pageNode)
lPageXmlDoc.append(
(pageXmlDoc, page.getAttribute('imageFilename')))
return lPageXmlDoc
def run(self):
"""
take a set of line in a page and mine it
"""
# use the lite version
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,listPages=range(self.firstPage,self.lastPage+1))
self.lPages= self.ODoc.getPages()
if self.bManual:
self.processWithTemplate(self.manualPattern,self.lPages)
else:
self.mainLineMining(self.lPages)
# lRes = self.mineLineFeature(self.lPages)
# print lRes
# returns the hierarchical set of elements (a list)
# for page , region, tree in lRes:
# self.tagDom(page, region)
# return
self.addTagProcessToMetadata(self.doc)
return self.doc
def run(self, doc):
"""
main issue: how to select the template: to be done by CVL
assuming IE and htr info are stored in the template
"""
# self.firstPage = 117
# self.lastPage= 118
if self.page2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
if self.recordType == 'D':
record = deathRecord(self.sModelName, self.sModelDir)
foo = self.processDeathWithTemplate
elif self.recordType == 'W':
record = weddingRecord(self.sModelName, self.sModelDir)
foo = self.processWeddingWithTemplate
elif self.recordType == 'B':
record = birthRecord(self.sModelName, self.sModelDir)
foo = self.processBirthWithTemplate
else:
print(f'record type not supported: {self.recordType}')
for page in self.lPages:
print("page: ", page.getNumber())
# self.testGTText(page)
# continue
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
if table.getNbRows() < 2:
if self.bDebug:
print("page: %s : not a table? %d/%d" %
(page.getNumber(), table.getNbRows(),
table.getNbColumns()))
continue
if self.BuseStoredTemplate:
if self.bDebug:
print("page: %s : table %d/%d" %
(page.getNumber(), table.getNbRows(),
table.getNbColumns()))
foo(table, record)
else:
self.mineTable(table, record)
self.evalData = record.generateOutput(self.evalData)
def createRefPerPage(self, doc):
"""
create a ref file from the xml one
for DAS 2018: one ref per graph(page)
"""
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(doc,
listPages=range(self.firstPage,
self.lastPage + 1))
dRows = {}
for page in self.ODoc.getPages():
#imageFilename="..\col\30275\S_Freyung_021_0001.jpg" width="977.52" height="780.0">
pageNode = etree.Element('PAGE')
# pageNode.set("number",page.getAttribute('number'))
#SINGLER PAGE pnum=1
pageNode.set("number", '1')
pageNode.set("imageFilename", page.getAttribute('imageFilename'))
pageNode.set("width", page.getAttribute('width'))
pageNode.set("height", page.getAttribute('height'))
root = etree.Element("DOCUMENT")
refdoc = etree.ElementTree(root)
root.append(pageNode)
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
tableNode = etree.Element('TABLE')
tableNode.set("x", table.getAttribute('x'))
tableNode.set("y", table.getAttribute('y'))
tableNode.set("width", table.getAttribute('width'))
tableNode.set("height", table.getAttribute('height'))
pageNode.append(tableNode)
for cell in table.getAllNamedObjects(XMLDSTABLECELLClass):
try:
dRows[int(cell.getAttribute("row"))].append(cell)
except KeyError:
dRows[int(cell.getAttribute("row"))] = [cell]
lYcuts = []
for rowid in sorted(dRows.keys()):
# print rowid, min(map(lambda x:x.getY(),dRows[rowid]))
lYcuts.append(
min(list(map(lambda x: x.getY(), dRows[rowid]))))
self.createRowsWithCuts(lYcuts, table, tableNode)
self.outputFileName = os.path.basename(
page.getAttribute('imageFilename')[:-3] + 'ref')
print(self.outputFileName)
self.writeDom(refdoc, bIndent=True)
return refdoc
def run(self, doc):
"""
main issue: how to select the template: to be done by CVL
assuming IE and htr info are stored in the template
"""
# self.firstPage = 9
# self.lastPage= 9
if self.page2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
dr = deathRecord(self.sModelName, self.sModelDir)
## selection of the templates first with X tables
###
for page in self.lPages:
print("page: ", page.getNumber())
# self.testGTText(page)
# continue
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
if table.getNbRows() < 2:
print("page: %s : not a table? %d/%d" %
(page.getNumber(), table.getNbRows(),
table.getNbColumns()))
continue
if self.BuseStoredTemplate:
# self.processWithTemplate(table, dr)
try:
self.processWithTemplate(table, dr)
except:
print('issue with page %s' % page)
else:
self.mineTable(table, dr)
self.evalData = dr.generateOutput(self.evalData)
def run(self, doc):
"""
load dom and find rows
"""
# conver to DS if needed
if self.bCreateRef:
if self.do2DS:
dsconv = primaAnalysis()
doc = dsconv.convert2DS(doc, self.docid)
refdoc = self.createRef(doc)
return refdoc
# single ref per page
refdoc = self.createRefPerPage(doc)
return None
if self.do2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
self.checkInputFormat(self.lPages)
self.findColumnsInDoc(self.lPages)
if self.bMining:
self.documentMining(self.lPages)
if self.bCreateRef:
refdoc = self.createRef(self.doc)
return refdoc
# if self.do2DS:
# # bakc to PageXml
# conv= DS2PageXMLConvertor()
# lPageXDoc = conv.run(self.doc)
# conv.storeMultiPageXml(lPageXDoc,self.getOutputFileName())
# print self.getOutputFileName()
# return None
return self.doc
def createRef(self, doc):
"""
create a ref file from the xml one
"""
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(doc,
listPages=range(self.firstPage,
self.lastPage + 1))
root = etree.Element("DOCUMENT")
refdoc = etree.ElementTree(root)
for page in self.ODoc.getPages():
#imageFilename="..\col\30275\S_Freyung_021_0001.jpg" width="977.52" height="780.0">
pageNode = etree.Element('PAGE')
pageNode.set("number", page.getAttribute('number'))
pageNode.set("pagekey",
os.path.basename(page.getAttribute('imageFilename')))
pageNode.set("width", page.getAttribute('width'))
pageNode.set("height", page.getAttribute('height'))
root.append(pageNode)
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
dCol = {}
tableNode = etree.Element('TABLE')
tableNode.set("x", table.getAttribute('x'))
tableNode.set("y", table.getAttribute('y'))
tableNode.set("width", table.getAttribute('width'))
tableNode.set("height", table.getAttribute('height'))
pageNode.append(tableNode)
for cell in table.getAllNamedObjects(XMLDSTABLECELLClass):
try:
dCol[int(cell.getAttribute("col"))].append(cell)
except KeyError:
dCol[int(cell.getAttribute("col"))] = [cell]
lXcuts = []
for colid in sorted(dCol.keys()):
lXcuts.append(
min(list(map(lambda x: x.getX(), dCol[colid]))))
self.createColumnsWithCuts(lXcuts, table, tableNode)
return refdoc
def run(self, doc):
"""
load dom and find rows
"""
# conver to DS if needed
if self.bCreateRef:
if self.do2DS:
dsconv = primaAnalysis()
doc = dsconv.convert2DS(doc, self.docid)
refdoc = self.createRef(doc)
return refdoc
# single ref per page
refdoc = self.createRefPerPage(doc)
return None
if self.do2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
# self.ODoc.loadFromDom(self.doc,listPages = range(30,31))
self.findColumnsInDoc(self.ODoc)
refdoc = self.createRef(self.doc)
# print refdoc.serialize('utf-8', 1)
if self.do2DS:
# bakc to PageXml
conv = DS2PageXMLConvertor()
lPageXDoc = conv.run(self.doc)
conv.storeMultiPageXml(lPageXDoc, self.getOutputFileName())
# print self.getOutputFileName()
return None
return self.doc
def processDocument(self, coldir, colid, docid, dom=None):
"""
process a single document
1 python ../../src/xml_formats/PageXml.py trnskrbs_5400/col/17442 --ext=pxml
2 python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form
3 python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400
4 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
5 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml
6 python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442
7 python ../../../TranskribusPyClient/src/TranskribusCommands/do_htrRnn.py <model-name> <dictionary-name> 5400 17442
wait
8 python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force
#covnert to ds
9 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
10 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml --doie --usetemplate
"""
#create Transkribus client
self.myTrKCient = TranskribusClient(sServerUrl=self.server,
proxies={},
loggingLevel=logging.WARN)
#login
_ = self.login(self.myTrKCient, trace=trace, traceln=traceln)
# self.downloadCollection(colid,coldir,docid,bNoImg=False,bForce=True)
## load dom
if dom is None:
self.inputFileName = os.path.abspath(
os.path.join(coldir, TableProcessing.sCOL,
docid + TableProcessing.sMPXMLExtension))
mpxml_doc = self.loadDom()
nbPages = MultiPageXml.getNBPages(mpxml_doc)
else:
# load provided mpxml
mpxml_doc = dom
nbPages = MultiPageXml.getNBPages(mpxml_doc)
# ### table registration: need to compute/select??? the template
# # perform LA separator, table registration, baseline with normalization
# #python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form
# tableregtool= LAProcessor()
# # latool.setParams(dParams)
# tableregtool.coldir = coldir
# tableregtool.docid = docid
# tableregtool.bTemplate, tableregtool.bSeparator , tableregtool.bBaseLine , tableregtool.bRegularTextLine = True,False,False,False
# # creates xml and a new mpxml
# mpxml_doc,nbPages = tableregtool.performLA(mpxml_doc)
#
#
# self.upLoadDocument(colid, coldir,docid,sNote='NLE workflow;table reg done')
lJobIDs = self.applyLA_URO(colid, docid, nbPages)
return
bWait = True
assert lJobIDs != []
jobid = lJobIDs[-1]
traceln("waiting for job %s" % jobid)
while bWait:
dInfo = self.myTrKCient.getJobStatus(jobid)
bWait = dInfo['state'] not in ['FINISHED', 'FAILED']
## coldir???
self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)
##STOP HERE FOR DAS newx testset:
return
# tag text for BIES cell
#python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400
"""
needed : doer = DU_ABPTable_TypedCRF(sModelName, sModelDir,
"""
doer = DU_ABPTable_TypedCRF(self.sRowModelName, self.sRowModelDir)
doer.load()
## needed predict at file level, and do not store dom, but return it
rowpath = os.path.join(coldir, "col")
BIESFiles = doer.predict([rowpath], docid)
BIESDom = self.loadDom(BIESFiles[0])
# res= BIESDom.saveFormatFileEnc('test.mpxml', "UTF-8",True)
# MPXML2DS
#python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
dsconv = primaAnalysis()
DSBIESdoc = dsconv.convert2DS(BIESDom, self.docid)
# create XMLDOC object
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(
DSBIESdoc) #,listPages = range(self.firstPage,self.lastPage+1))
# create row
#python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml
rdc = RowDetection()
rdc.findRowsInDoc(self.ODoc)
#python ../../src/xml_formats/DS2PageXml.py -i trnskrbs_5400/out/17442.ds_xml --multi
# DS2MPXML
DS2MPXML = DS2PageXMLConvertor()
lPageXml = DS2MPXML.run(self.ODoc.getDom())
if lPageXml != []:
# if DS2MPXML.bMultiPages:
newDoc = MultiPageXml.makeMultiPageXmlMemory(
map(lambda xy: xy[0], lPageXml))
outputFileName = os.path.join(
self.coldir, sCOL,
self.docid + TableProcessing.sMPXMLExtension)
newDoc.write(outputFileName,
xml_declaration=True,
encoding="UTF-8",
pretty_print=True)
# else:
# DS2MPXML.storePageXmlSetofFiles(lPageXml)
return
#upload
# python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442
self.upLoadDocument(colid,
coldir,
docid,
sNote='NLE workflow;table row done')
## apply HTR
## how to deal with specific dictionaries?
## here need to know the ontology and the template
nbPages = 1
jobid = self.applyHTR(colid, docid, nbPages, self.sHTRmodel,
self.sDictName)
bWait = True
traceln("waiting for job %s" % jobid)
while bWait:
dInfo = self.myTrKCient.getJobStatus(jobid)
bWait = dInfo['state'] not in ['FINISHED', 'FAILED', 'CANCELED']
# download where???
# python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force
# coldir is not right!! coldir must refer to the parent folder!
self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)
class lineMiner(Component.Component):
"""
lineMiner class: a component to mine column-like page layout
"""
#DEFINE the version, usage and description of this particular component
usage = ""
version = "v.01"
description = "description: line miner "
#--- INIT -------------------------------------------------------------------------------------------------------------
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "lineMiner", self.usage, self.version, self.description)
# TH for comparing numerical features for X
self.THNUMERICAL= 20
# use for evaluation
self.THCOMP = 10
self.evalData= None
self.bManual = False
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
if "pattern" in dParams:
self.manualPattern = eval( dParams["pattern"])
self.bManual=True
def mainLineMining(self,lPages):
"""
mine with incremental length
"""
import util.TwoDNeighbourhood as TwoDRel
lVEdge = []
lLElts=[ [] for i in range(0,len(lPages))]
for i,page in enumerate(lPages):
lElts= page.getAllNamedObjects(XMLDSTEXTClass)
for e in lElts:
e.lnext=[]
## filter elements!!!
# lElts = filter(lambda x:min(x.getHeight(),x.getWidth()) > 10,lElts)
# lElts = filter(lambda x:x.getHeight() > 10,lElts)
# lElts = list(filter(lambda x:x.getX() > 60,lElts))
lElts.sort(key=lambda x:x.getY())
lLElts[i]=lElts
lVEdge = TwoDRel.findVerticalNeighborEdges(lElts)
for a,b in lVEdge:
a.lnext.append( b )
for i,page, in enumerate(lPages):
lElts= lLElts[i]
for elt in lElts:
### need of more abstract features: justified, center, left, right + numerical
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['x','2','xc','text'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
seqGen = sequenceMiner()
seqGen.setMaxSequenceLength(1)
seqGen.setSDC(0.7) # related to noise level AND STRUCTURES (if many columns)
_ = seqGen.featureGeneration(lElts,2)
seqGen.setObjectLevel(XMLDSTEXTClass)
# for registration: needs to be replaced by ._lRegValues
print("sequence of elements and their features:")
for elt in lElts:
elt.lFeatureForParsing=elt.getSetofFeatures()
# print (elt, elt.lFeatureForParsing)
lTerminalTemplates=[]
lCurList=lElts[:]
for iLen in range(1,3):
lCurList,lTerminalTemplates = self.mineLineFeature(seqGen,lCurList,lTerminalTemplates,iLen)
del seqGen
def mineLineFeature(self,seqGen,lCurList,lTerminalTemplates,iLen):
"""
get a set of lines and mine them
"""
seqGen.setMinSequenceLength(iLen)
seqGen.setMaxSequenceLength(iLen)
print( '***'*20, iLen)
seqGen.bDebug = False
for elt in lCurList:
if elt.getSetofFeatures() is None:
elt.resetFeatures()
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['virtual'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
elt.lFeatureForParsing=elt.getSetofFeatures()
else:
elt.setSequenceOfFeatures(elt.lFeatureForParsing)
# print elt, elt.getSetofFeatures()
lSortedFeatures = seqGen.featureGeneration(lCurList,2)
for cf in lSortedFeatures:
cf.setWeight(len((cf.getNodes())))
# for x in lCurList:
# print( x, x.getCanonicalFeatures())
lmaxSequence = seqGen.generateItemsets(lCurList)
seqGen.bDebug = False
lSeq, _ = seqGen.generateMSPSData(lmaxSequence,lSortedFeatures + lTerminalTemplates,mis = 0.01)
lPatterns = seqGen.miningSequencePrefixScan(lSeq)
# model = PrefixSpan.train(lSeq, minSupport = 0.1, maxPatternLength = 10)
# result = model.freqSequences().collect()
# result.sort(key=lambda x:x.freq)
# lPatterns=[]
# for fs in result:
# if fs.freq > 1:
# # for i in fs.sequence:
# # for x in i:
# # print(x,x.getValue())
# # i.sort(key=lambda x:x.getValue())
# lPatterns.append((fs.sequence,fs.freq))
# lPatterns = list(filter(lambda p_s:len(p_s[0]) >= seqGen.getMinSequenceLength() and len(p_s[0]) <= seqGen.getMaxSequenceLength(),lPatterns))
# lPatterns = seqGen.beginMiningSequences(lSeq,lSortedFeatures,lMIS)
if lPatterns is None:
return lCurList, lTerminalTemplates
lPatterns.sort(key=lambda xy:xy[1], reverse=True)
print ("List of patterns and their support:")
for p,support in lPatterns:
if support >= 1:
print (p, support)
seqGen.THRULES = 0.95
lSeqRules = seqGen.generateSequentialRules(lPatterns)
" here store features which are redundant and consider only the core feature"
_,dCP = self.getPatternGraph(lSeqRules)
dTemplatesCnd = self.analyzeListOfPatterns(lPatterns,dCP)
lFullTemplates,lTerminalTemplates,tranprob = seqGen.testTreeKleeneageTemplates(dTemplatesCnd, lCurList)
## here we have a graph; second : is it useful here to correct noise??
## allows for selecting templates ?
self.selectFinalTemplates(lTerminalTemplates,tranprob,lCurList)
## store parsed sequences in mytemplate
### patterns are competing: generate a set of parsing ???
for mytemplate in lFullTemplates[:1]: # dTemplatesCnd.keys():
# for _,_, mytemplate in dTemplatesCnd[templateType][:1]:
mytemplate.print_()
# print '___'*30
# print lCurList
isKleenePlus,_,lCurList = seqGen.parseWithTreeTemplate(mytemplate,lCurList,bReplace=True)
for elt in lCurList:
if elt.getSetofFeatures() is None:
elt.resetFeatures()
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['virtual'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
elt.lFeatureForParsing=elt.getSetofFeatures()
else:
elt.setSequenceOfFeatures(elt.lFeatureForParsing)
# self.printTreeView(lCurList)
# print 'curList:',lCurList
# print len(lCurList)
print( "final hierarchy")
self.printTreeView(lCurList)
# lRegions= self.getRegionsFromStructure(page,lCurList)
# store all interation
# lPageRegions.append((page,lRegions,lCurList))
return lCurList, lTerminalTemplates
def mineLineFeatureBasic(self,lPages):
"""
get a set of lines and mine them
"""
import util.TwoDNeighbourhood as TwoDRel
lPageRegions = []
lVEdge = []
lLElts=[ [] for i in range(0,len(lPages))]
for i,page in enumerate(lPages):
lElts= page.getAllNamedObjects(XMLDSTEXTClass)
for e in lElts:
e.lnext=[]
## filter elements!!!
lElts = list(filter(lambda x:min(x.getHeight(),x.getWidth()) > 10,lElts))
# lElts = filter(lambda x:x.getHeight() > 10,lElts)
# lElts = list(filter(lambda x:x.getX() > 100,lElts))
lElts.sort(key=lambda x:x.getY())
lLElts[i]=lElts
lVEdge = TwoDRel.findVerticalNeighborEdges(lElts)
for a,b in lVEdge:
a.lnext.append( b )
for i,page, in enumerate(lPages):
lElts= lLElts[i]
for elt in lElts:
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['x','x2','xc','text'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
# print elt.getSetofFeatures()
seqGen = sequenceMiner()
seqGen.setMaxSequenceLength(1)
seqGen.setSDC(0.5) # related to noise level AND STRUCTURES (if many columns)
lSortedFeatures = seqGen.featureGeneration(lElts,2)
seqGen.setObjectLevel(XMLDSTEXTClass)
# for registration: needs to be replaced by ._lRegValues
print( "sequence of elements and their features:")
for elt in lElts:
elt.lFeatureForParsing=elt.getSetofFeatures()
print( elt, elt.lFeatureForParsing)
icpt=0
lTerminalTemplates = []
lCurList= lElts
# print len(lCurList)
while icpt < 2:
# seqGen.bDebug = False
## generate sequences
if icpt > 0:
seqGen.setMinSequenceLength(2)
seqGen.setMaxSequenceLength(2)
print( '***'*20, icpt)
seqGen.bDebug = False
for elt in lCurList:
if elt.getSetofFeatures() is None:
elt.resetFeatures()
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['virtual'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
elt.lFeatureForParsing=elt.getSetofFeatures()
else:
elt.setSequenceOfFeatures(elt.lFeatureForParsing)
lSortedFeatures = seqGen.featureGeneration(lCurList,2)
lmaxSequence = seqGen.generateItemsets(lCurList)
seqGen.bDebug = False
lSeq, lMIS = seqGen.generateMSPSData(lmaxSequence,lSortedFeatures + lTerminalTemplates,mis = 0.2)
lPatterns = seqGen.miningSequencePrefixScan(lSeq)
if lPatterns is None:
return None
# lPatterns = seqGen.beginMiningSequences(lSeq,lSortedFeatures,lMIS)
lPatterns.sort(key=lambda xy:xy[1], reverse=True)
print( "List of patterns and their support:")
for p,support in lPatterns:
if support >= 1:
print (p, support)
seqGen.THRULES = 0.95
lSeqRules = seqGen.generateSequentialRules(lPatterns)
" here store features which are redundant and consider only the core feature"
_,dCP = self.getPatternGraph(lSeqRules)
dTemplatesCnd = self.analyzeListOfPatterns(lPatterns,dCP,icpt)
lFullTemplates,lTerminalTemplates,tranprob = seqGen.testTreeKleeneageTemplates(dTemplatesCnd, lCurList)
# lFullTemplates = seqGen.testTreeKleeneageTemplates(dTemplatesCnd, lCurList)
# print tranprob
# print lTerminalTemplates
## here we have a graph; second : is it useful here to correct noise??
## allows for selecting templates ?
self.selectFinalTemplates(lTerminalTemplates,tranprob,lCurList)
## store parsed sequences in mytemplate
### patterns are competing: generate a set of parsing ???
for mytemplate in lFullTemplates[:10]: # dTemplatesCnd.keys():
# for _,_, mytemplate in dTemplatesCnd[templateType][:1]:
mytemplate.print_()
# print '___'*30
# print lCurList
isKleenePlus,_,lCurList = seqGen.parseWithTreeTemplate(mytemplate,lCurList,bReplace=True)
for elt in lCurList:
if elt.getSetofFeatures() is None:
elt.resetFeatures()
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['virtual'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
elt.lFeatureForParsing = elt.getSetofFeatures()
else:
elt.setSequenceOfFeatures(elt.lFeatureForParsing)
# self.printTreeView(lCurList)
icpt +=1
# print 'curList:',lCurList
# print len(lCurList)
print ("final hierarchy")
self.printTreeView(lCurList)
# lRegions= self.getRegionsFromStructure(page,lCurList)
# store all interation
# lPageRegions.append((page,lRegions,lCurList))
return lPageRegions
def selectFinalTemplates(self,lTemplates,transProb,lElts):
"""
apply viterbi to select best sequence of templates
"""
import spm.viterbi as viterbi
if lTemplates == []:
return None
def buildObs(lTemplates,lElts):
"""
build observation prob
"""
N = len(lTemplates) + 1
obs = np.zeros((N,len(lElts)), dtype=np.float16) +10e-3
for i,temp in enumerate(lTemplates):
for j,elt in enumerate(lElts):
# how to dela with virtual nodes
try:
_, _, score= temp.registration(elt)
except : score =1
if score == -1:
score= 0.0
obs[i,j]= score
if np.isinf(obs[i,j]):
obs[i,j] = 64000
if np.isnan(obs[i,j]):
obs[i,j] = 0.0
# print i,j,elt,elt.lX, temp,score
#add no-template:-1
return obs / np.amax(obs)
N= len(lTemplates) + 1
initialProb = np.ones(N)
initialProb = np.reshape(initialProb,(N,1))
obs = buildObs(lTemplates,lElts)
np.set_printoptions(precision= 3, linewidth =1000)
# print "transProb"
# print transProb
# print
# print obs
d = viterbi.Decoder(initialProb, transProb, obs)
states,score = d.Decode(np.arange(len(lElts)))
# add empty template (last one in state)
lTemplates.append(None)
print (states, score)
#assign to each elt the template assigned by viterbi
for i,elt, in enumerate(lElts):
# try: print elt,elt.lX, lTemplates[states[i]]
# except: print elt, elt.lX, 'no template'
mytemplate= lTemplates[states[i]]
elt.resetTemplate()
if mytemplate is not None:
elt.addTemplate(mytemplate)
try:
registeredPoints, lMissing, score= mytemplate.registration(elt)
except:
registeredPoints = None
if registeredPoints:
# print registeredPoints, lMissing , score
if lMissing != []:
registeredPoints.extend(zip(lMissing,lMissing))
registeredPoints.sort(key=lambda xy:xy[1].getValue())
lcuts = map(lambda refcut:refcut[1],registeredPoints)
## store features for the final parsing!!!
# print elt, lcuts
# elt.addVSeparator(mytemplate,lcuts)
# return the new list with kleenePlus elts for next iteration
## reparse ?? YES using the featureSet given by viterbi -> create an objectClass per kleenePlus element: objects: sub tree
# print lTemplates[0]
# self.parseWithTemplate(lTemplates[0], lElts)
# elt = template
return score
def createItemSetFromNext(self,lElts,iLen):
"""
create itemset of length iLen using .lnext structures
"""
def getKleenePlusFeatures(self,lElts):
"""
select KleenePlus elements based on .next (only possible for unigrams)
"""
dFreqFeatures={}
dKleenePlusFeatures = {}
lKleenePlus=[]
for elt in lElts:
for fea in elt.getSetofFeatures():
try:dFreqFeatures[fea] +=1
except KeyError:dFreqFeatures[fea] = 1
for nextE in elt.next:
if fea in nextE.getSetofFeatures():
try:dKleenePlusFeatures[fea].append((elt,nextE))
except KeyError:dKleenePlusFeatures[fea]=[(elt,nextE)]
for fea in dFreqFeatures:
try:
dKleenePlusFeatures[fea]
if len(dKleenePlusFeatures[fea]) >= 0.5 * dFreqFeatures[fea]:
lKleenePlus.append(fea)
except:
pass
return lKleenePlus
def computePatternScore(self,pattern):
"""
consider the frequency of the pattern and the weights of the features
"""
fScore = 0
#terminal
if not isinstance(pattern, list):
fScore += pattern.getCanonical().getWeight()
else:
for child in pattern:
fScore += self.computePatternScore(child)
# print 'score:',pattern ,fScore
return fScore
def analyzeListOfPatterns(self,lPatterns,dCA,):
"""
select patterns with no ancestor
other criteria ?
if many with similar frequency: sort using computePatternScore?
"""
# reorder lPatterns considering feature weights and # of elements (for equally frequent patterns)
# lPatterns.sort(key=lambda (x,y):self.computePatternScore(x),reverse=True)
# for x,y in lPatterns:
# print x,y,self.computePatternScore(x)
dTemplatesTypes = {}
for pattern,support in filter(lambda xy:xy[1]>1,lPatterns):
try:
dCA[str(pattern)]
bSkip = True
except KeyError:bSkip=False
# if step > 0 and len(pattern) == 1:
# bSkip=True
if not bSkip:
template = treeTemplateClass()
template.setPattern(pattern)
template.buildTreeFromPattern(pattern)
template.setType('lineTemplate')
try:dTemplatesTypes[template.__class__.__name__].append((pattern, support, template))
except KeyError: dTemplatesTypes[template.__class__.__name__] = [(pattern,support,template)]
return dTemplatesTypes
def processWithTemplate(self,lPattern,lPages):
"""
apply a known pattern
"""
def convertStringtoPattern(xcur):
## need to integrate the 'virtual level'
lRes=[]
for elt in xcur:
if isinstance(elt,list):
lRes.extend([convertStringtoPattern(elt)])
else:
try:
float(elt)
f= featureObject()
f.setName("x")
f.setType(featureObject.NUMERICAL)
f.setValue(elt)
f.setObjectName(elt)
f.setWeight(1)
f.setTH(self.THNUMERICAL)
except:
f= featureObject()
f.setName("f")
f.setType(featureObject.EDITDISTANCE)
f.setValue(elt)
f.setTH(100.0)
lRes.append(f)
return lRes
lfPattern =convertStringtoPattern(lPattern)
# print lfPattern
# create a template from lfPattern!
mytemplate = treeTemplateClass()
# mytemplate.setPattern(lfPattern)
mytemplate.buildTreeFromPattern(lfPattern)
mytemplate.print_()
for page in lPages:
seqGen = sequenceMiner()
seqGen.setObjectLevel(XMLDSTEXTClass)
lElts= page.getAllNamedObjects(XMLDSTEXTClass)
for elt in lElts:
elt.setFeatureFunction(elt.getSetOfListedAttributes,self.THNUMERICAL,lFeatureList=['x','x2','text'],myLevel=XMLDSTEXTClass)
elt.computeSetofFeatures()
elt.lFeatureForParsing=elt.getSetofFeatures()
lParsing,lNewSeq = seqGen.parseWithTreeTemplate(mytemplate,lElts,bReplace=True)
del seqGen
self.printTreeView(lNewSeq)
# process lNewSeq: create the output data structure?
def printTreeView(self,lElts,level=0):
"""
move to structuralMining?
"""
for elt in lElts:
if elt.getAttribute('virtual'):
print (" "*level, 'Node', elt.getAttribute('virtual'))
self.printTreeView(elt.getObjects(),level+1)
else:
print (" " * level,elt.getContent())
# try:print (" "*level, elt, elt.getContent())
# except: print (elt._content[:3])
def getPatternGraph(self,lRules):
"""
create an graph which linsk exoannded patterns
(a) -> (ab)
(abc) -> (abcd)
rule = (newPattern,item,i,pattern, fConfidence)
RULE: [['x=19.0', 'x=48.0', 'x=345.0'], ['x=19.0', 'x=126.0', 'x=345.0']] => 'x=464.0'[0] (22.0/19.0 = 0.863636363636)
can be used for tagging go up unitl no paretn
"""
dParentChild= {}
dChildParent= {}
for lhs, rhs, itemsetIndex, fullpattern, fConfidence in lRules:
try:dParentChild[str(fullpattern)].append(lhs)
except KeyError:dParentChild[str(fullpattern)] = [lhs]
try:dChildParent[str(lhs)].append(fullpattern)
except KeyError:dChildParent[str(lhs)] = [fullpattern]
# # for bigram: extend to grammy
# for child in dChildParent.keys():
# ltmp=[]
# if len(eval(child)) == 2:
# for parent in dChildParent[child]:
# try:
# ltmp.extend(dChildParent[str(parent)])
# except KeyError:pass
# dChildParent[child].extend(ltmp)
return dParentChild, dChildParent
def generateTestOutput(self,lPages):
"""
create a run XML file
"""
root = etree.Element('DOCUMENT')
self.evalData = etree.ElementTree(root)
for page in lPages:
domp = etree.Element('PAGE')
domp.set('number',page.getAttribute('number'))
root.append(domp)
for sep in page.lVSeparator:
# print( page.lVSeparator)
domsep= etree.Element('SeparatorRegion')
domp.append(domsep)
domsep.set('x', str(sep[0]))
return self.evalData
def getRegionsFromStructure(self,page,lTree):
"""
tag the dom with what?
"""
lZone=[]
srx, sry = 9e9, 9e9
srx2,sry2 = 0 , 0
lSubZone=[]
for elt in lTree:
if elt.getAttribute('virtual'):
rx,ry,rx2,ry2,lsub = self.getRegionsFromStructure(page,elt.getObjects())
srx = min(srx,rx)
sry = min(sry,ry)
srx2= max(srx2,rx2)
sry2= max(sry2,ry2)
lSubZone.append([rx,ry,rx2,ry2,lsub])
else:
lZone.append(elt)
# get BB of the zone
fMinX , fMinY = srx , sry
fMaxX , fMaxY = srx2 , sry2
for e in lZone:
if e.getX2() > fMaxX:
fMaxX= e.getX2()
if e.getY2() > fMaxY:
fMaxY= e.getY2()
if e.getX() < fMinX:
fMinX= e.getX()
if e.getY() < fMinY:
fMinY= e.getY()
# has substructure
if srx != 9e9:
return [ fMinX,fMinY,fMaxX,fMaxY , lSubZone]
else:
return [ fMinX,fMinY,fMaxX,fMaxY, []]
def tagDom(self,page,region):
"""
tag page with region (x,y,x2,y2)
"""
fMinX , fMinY, fMaxX , fMaxY, ltail = region
# new region node
regionNode = etree.Element('REGION')
page.getNode().append(regionNode)
regionNode.set('x',str(fMinX))
regionNode.set('y',str(fMinY))
regionNode.set('height',str(fMaxY-fMinY))
regionNode.set('width',str(fMaxX - fMinX))
print()
print (region)
print (regionNode)
[self.tagDom(page,tail) for tail in ltail]
return regionNode
#--- RUN ---------------------------------------------------------------------------------------------------------------
def run(self):
"""
take a set of line in a page and mine it
"""
# use the lite version
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,listPages=range(self.firstPage,self.lastPage+1))
self.lPages= self.ODoc.getPages()
if self.bManual:
self.processWithTemplate(self.manualPattern,self.lPages)
else:
self.mainLineMining(self.lPages)
# lRes = self.mineLineFeature(self.lPages)
# print lRes
# returns the hierarchical set of elements (a list)
# for page , region, tree in lRes:
# self.tagDom(page, region)
# return
self.addTagProcessToMetadata(self.doc)
return self.doc
#--- TESTS -------------------------------------------------------------------------------------------------------------
#
# Here we have the code used to test this component on a prepared testset (see under <ROOT>/test/common)
# Do: python ../../src/common/TypicalComponent.py --test REF_TypicalComponent/
#
def testeval(self,srefData,srunData, bVisual):
"""
Test found reftemplate and reftemplate
"""
cntOk = cntErr = cntMissed = 0
RefData = etree.XML(srefData.strip("\n").encode('utf-8'))
RunData = etree.XML(srunData.strip("\n").encode('utf-8'))
lRun = []
if RunData is not None:
lpages = RunData.xpath('//%s' % ('PAGE'))
for page in lpages:
if page.get('reftemplate'):
lRun.append(eval(page.get('reftemplate')))
lRef = []
lPages = RefData.xpath('//%s' % ('PAGE'))
for page in lPages:
if page.get('reftemplate'):
lRef.append(eval(page.get('reftemplate')))
else: lRef.append([])
ltisRefsRunbErrbMiss= list()
for i in range(0,len(lRef)):
lRefCovered = []
if lRun[i] ==[]:
runLen=0
else:
posrun = lRun[i][0]
runLen = len(lRun[i][posrun+1])
if lRef[i]==[]:
refLen=0
refElt=None
posref=None
else:
posref=lRef[i][0]
refLen= len(lRef[i][posref+1])
curRun = curRef = 0
while curRun <= runLen - 1: # or curRef <= refLen -1:
bErr, bMiss = False, False
try:
runElt = lRun[i][posrun+1][curRun]
except IndexError: runElt = None
# print '___',curRun,runElt
curRef = 0
bFound = False
while not bFound and curRef <= refLen - 1:
try: refElt = lRef[i][posref+1][curRef]
except IndexError: refElt = None
# self.compareString(runElt,runElt)
if runElt and refElt not in lRefCovered and self.testComparePageVertical(runElt, refElt):
bFound = True
lRefCovered.append(refElt)
resRef=refElt
else:
curRef += 1
if bFound:
if bVisual:print ("FOUND:", runElt, ' -- ', lRefCovered[-1])
cntOk += 1
curRun += 1
else:
resRef=''
curRun += 1
cntErr += 1
bErr = True
# bMiss = True
if bVisual:print ("ERROR:", runElt)
ltisRefsRunbErrbMiss.append( (i, resRef, runElt,bErr, bMiss) )
if posref is not None:
for ref in lRef[i][posref+1]:
if ref not in lRefCovered:
ltisRefsRunbErrbMiss.append( (i, ref, '',False, True) )
# add missed elements!
# print 'missed', len(lRef[i][posref+1]) , len(lRefCovered), lRef[i][posref+1], lRefCovered
cntMissed += 1#(len(lRef[i][posref+1]) - len(lRefCovered))
ltisRefsRunbErrbMiss.sort(key=lambda x_y_z_t_u:x_y_z_t_u[0])
return (cntOk, cntErr, cntMissed,ltisRefsRunbErrbMiss)
def testRun(self, filename, outFile=None):
"""
testRun is responsible for running the component on this file and returning a string that reflects the result in a way
that is understandable to a human and to a program. Nicely serialized Python data or XML is fine
"""
doc = self.loadDom(filename)
self.doc= doc
self.run()
# doc.freeDoc()
self.generateTestOutput(self.lPages)
if outFile: self.writeDom(doc)
return etree.tostring(self.evalData,encoding='unicode')
def testCompare(self, srefData, srunData, bVisual=False):
"""
Our comparison is very simple: same or different. N
We anyway return this in term of precision/recall
If we want to compute the error differently, we must define out own testInit testRecord, testReport
"""
dicTestByTask = dict()
dicTestByTask['EVAL']= self.testeval(srefData, srunData,bVisual)
return dicTestByTask
class columnDetection(Component.Component):
"""
build table column
"""
usage = ""
version = "v.01"
description = "description: column Detection"
#--- INIT -------------------------------------------------------------------------------------------------------------
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "columnDetection", self.usage,
self.version, self.description)
self.colname = None
self.docid = None
self.do2DS = False
# for --test
self.bCreateRef = False
self.evalData = None
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
# if dParams.has_key("coldir"):
# self.colname = dParams["coldir"]
if "docid" in dParams:
self.docid = dParams["docid"]
if "dsconv" in dParams:
self.do2DS = dParams["dsconv"]
if "createref" in dParams:
self.bCreateRef = dParams["createref"]
# def createCells(self, table):
# """
# create new cells using BIESO tags
# @input: tableObeject with old cells
# @return: tableObject with BIES cells
# @precondition: requires columns
#
# """
# for col in table.getColumns():
# lNewCells=[]
# # keep original positions
# col.resizeMe(XMLDSTABLECELLClass)
# for cell in col.getCells():
# # print cell
# curChunk=[]
# lChunks = []
# # print map(lambda x:x.getAttribute('type'),cell.getObjects())
# # print map(lambda x:x.getID(),cell.getObjects())
# cell.getObjects().sort(key=lambda x:x.getY())
# for txt in cell.getObjects():
# # print txt.getAttribute("type")
# if txt.getAttribute("type") == 'RS':
# if curChunk != []:
# lChunks.append(curChunk)
# curChunk=[]
# lChunks.append([txt])
# elif txt.getAttribute("type") in ['RI', 'RE']:
# curChunk.append(txt)
# elif txt.getAttribute("type") == 'RB':
# if curChunk != []:
# lChunks.append(curChunk)
# curChunk=[txt]
# elif txt.getAttribute("type") == 'RO':
# ## add Other as well???
# curChunk.append(txt)
#
# if curChunk != []:
# lChunks.append(curChunk)
#
# if lChunks != []:
# # create new cells
# table.delCell(cell)
# irow= cell.getIndex()[0]
# for i,c in enumerate(lChunks):
# # print map(lambda x:x.getAttribute('type'),c)
# #create a new cell per chunk and replace 'cell'
# newCell = XMLDSTABLECELLClass()
# newCell.setPage(cell.getPage())
# newCell.setParent(table)
# newCell.setName(ds_xml.sCELL)
# newCell.setIndex(irow+i,cell.getIndex()[1])
# newCell.setObjectsList(c)
# newCell.resizeMe(XMLDSTEXTClass)
# newCell.tagMe2()
# for o in newCell.getObjects():
# o.setParent(newCell)
# o.tagMe()
# # table.addCell(newCell)
# lNewCells.append(newCell)
# cell.getNode().getparent().remove(cell.getNode())
# del(cell)
# col.setObjectsList(lNewCells[:])
# [table.addCell(c) for c in lNewCells]
#
# # print col.tagMe()
def createTable(self, page):
"""
BB of all elements?
"""
def processPage(self, page):
from util.XYcut import mergeSegments
### skrinking to be done:
lCuts, x1, x2 = mergeSegments(
[(x.getX(), x.getX() + 20, x)
for x in page.getAllNamedObjects(XMLDSTEXTClass)], 0)
for x, y, cut in lCuts:
ll = list(cut)
ll.sort(key=lambda x: x.getY())
traceln(len(ll))
# traceln (list(map(lambda x:x.getContent(),ll)))
def findColumnsInDoc(self, ODoc):
"""
find columns for each table in ODoc
"""
self.lPages = ODoc.getPages()
# not always?
# self.mergeLineAndCells(self.lPages)
for page in self.lPages:
traceln("page: %d" % page.getNumber())
self.processPage(page)
def run(self, doc):
"""
load dom and find rows
"""
# conver to DS if needed
if self.bCreateRef:
if self.do2DS:
dsconv = primaAnalysis()
doc = dsconv.convert2DS(doc, self.docid)
refdoc = self.createRef(doc)
return refdoc
# single ref per page
refdoc = self.createRefPerPage(doc)
return None
if self.do2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
# self.ODoc.loadFromDom(self.doc,listPages = range(30,31))
self.findColumnsInDoc(self.ODoc)
refdoc = self.createRef(self.doc)
# print refdoc.serialize('utf-8', 1)
if self.do2DS:
# bakc to PageXml
conv = DS2PageXMLConvertor()
lPageXDoc = conv.run(self.doc)
conv.storeMultiPageXml(lPageXDoc, self.getOutputFileName())
# print self.getOutputFileName()
return None
return self.doc
################ TEST ##################
def testRun(self, filename, outFile=None):
"""
evaluate using ABP new table dataset with tablecell
"""
self.evalData = None
doc = self.loadDom(filename)
doc = self.run(doc)
self.evalData = self.createRef(doc)
if outFile: self.writeDom(doc)
# return self.evalData.serialize('utf-8',1)
return etree.tostring(self.evalData,
encoding='unicode',
pretty_print=True)
def overlapX(self, zone):
[a1, a2] = self.getX(), self.getX() + self.getWidth()
[b1, b2] = zone.getX(), zone.getX() + zone.getWidth()
return min(a2, b2) >= max(a1, b1)
def overlapY(self, zone):
[a1, a2] = self.getY(), self.getY() + self.getHeight()
[b1, b2] = zone.getY(), zone.getY() + zone.getHeight()
return min(a2, b2) >= max(a1, b1)
def signedRatioOverlap(self, z1, z2):
"""
overlap self and zone
return surface of self in zone
"""
[x1, y1, h1, w1] = z1.getX(), z1.getY(), z1.getHeight(), z1.getWidth()
[x2, y2, h2, w2] = z2.getX(), z2.getY(), z2.getHeight(), z2.getWidth()
fOverlap = 0.0
if self.overlapX(z2) and self.overlapY(z2):
[x11, y11, x12, y12] = [x1, y1, x1 + w1, y1 + h1]
[x21, y21, x22, y22] = [x2, y2, x2 + w2, y2 + h2]
s1 = w1 * h1
# possible ?
if s1 == 0: s1 = 1.0
#intersection
nx1 = max(x11, x21)
nx2 = min(x12, x22)
ny1 = max(y11, y21)
ny2 = min(y12, y22)
h = abs(nx2 - nx1)
w = abs(ny2 - ny1)
inter = h * w
if inter > 0:
fOverlap = inter / s1
else:
# if overX and Y this is not possible !
fOverlap = 0.0
return fOverlap
def findSignificantOverlap(self, TOverlap, ref, run):
"""
return
"""
pref, rowref = ref
prun, rowrun = run
if pref != prun: return False
return rowref.ratioOverlap(rowrun) >= TOverlap
def testCPOUM(self, TOverlap, srefData, srunData, bVisual=False):
"""
TOverlap: Threshols used for comparing two surfaces
Correct Detections:
under and over segmentation?
"""
cntOk = cntErr = cntMissed = 0
RefData = etree.XML(srefData.strip("\n").encode('utf-8'))
RunData = etree.XML(srunData.strip("\n").encode('utf-8'))
# try:
# RunData = libxml2.parseMemory(srunData.strip("\n"), len(srunData.strip("\n")))
# except:
# RunData = None
# return (cntOk, cntErr, cntMissed)
lRun = []
if RunData:
lpages = RunData.xpath('//%s' % ('PAGE'))
for page in lpages:
pnum = page.get('number')
#record level!
lRows = page.xpath(".//%s" % ("ROW"))
lORows = map(lambda x: XMLDSTABLEROWClass(0, x), lRows)
for row in lORows:
row.fromDom(row._domNode)
row.setIndex(row.getAttribute('id'))
lRun.append((pnum, row))
print(lRun)
lRef = []
lPages = RefData.xpath('//%s' % ('PAGE'))
for page in lPages:
pnum = page.get('number')
lRows = page.xpath(".//%s" % ("ROW"))
lORows = map(lambda x: XMLDSTABLEROWClass(0, x), lRows)
for row in lORows:
row.fromDom(row._domNode)
row.setIndex(row.getAttribute('id'))
lRef.append((pnum, row))
refLen = len(lRef)
# bVisual = True
ltisRefsRunbErrbMiss = list()
lRefCovered = []
for i in range(0, len(lRun)):
iRef = 0
bFound = False
bErr, bMiss = False, False
runElt = lRun[i]
# print '\t\t===',runElt
while not bFound and iRef <= refLen - 1:
curRef = lRef[iRef]
if runElt and curRef not in lRefCovered and self.findSignificantOverlap(
TOverlap, runElt, curRef):
bFound = True
lRefCovered.append(curRef)
iRef += 1
if bFound:
if bVisual: print("FOUND:", runElt, ' -- ', lRefCovered[-1])
cntOk += 1
else:
curRef = ''
cntErr += 1
bErr = True
if bVisual: print("ERROR:", runElt)
if bFound or bErr:
ltisRefsRunbErrbMiss.append(
(int(runElt[0]), curRef, runElt, bErr, bMiss))
for i, curRef in enumerate(lRef):
if curRef not in lRefCovered:
if bVisual: print("MISSED:", curRef)
ltisRefsRunbErrbMiss.append(
(int(curRef[0]), curRef, '', False, True))
cntMissed += 1
ltisRefsRunbErrbMiss.sort(key=lambda xyztu: xyztu[0])
# print cntOk, cntErr, cntMissed,ltisRefsRunbErrbMiss
return (cntOk, cntErr, cntMissed, ltisRefsRunbErrbMiss)
def testCompare(self, srefData, srunData, bVisual=False):
"""
as in Shahad et al, DAS 2010
Correct Detections
Partial Detections
Over-Segmented
Under-Segmented
Missed
False Positive
"""
dicTestByTask = dict()
dicTestByTask['T50'] = self.testCPOUM(0.50, srefData, srunData,
bVisual)
# dicTestByTask['T75']= self.testCPOUM(0.750,srefData,srunData,bVisual)
# dicTestByTask['T100']= self.testCPOUM(0.50,srefData,srunData,bVisual)
# dicTestByTask['FirstName']= self.testFirstNameRecord(srefData, srunData,bVisual)
# dicTestByTask['Year']= self.testYear(srefData, srunData,bVisual)
return dicTestByTask
def createColumnsWithCuts(self, lXCuts, table, tableNode, bTagDoc=False):
"""
create column dom node
"""
prevCut = None
lXCuts.sort()
for index, cut in enumerate(lXCuts):
# first correspond to the table: no rpw
if prevCut is not None:
colNode = etree.Element("COL")
tableNode.append(colNode)
colNode.set('x', str(prevCut))
colNode.set('width', "{:.2f}".format(cut - prevCut))
colNode.set('y', str(table.getY()))
colNode.set('height', str(table.getHeight()))
colNode.set('id', str(index - 1))
prevCut = cut
#last
cut = table.getX2()
colNode = etree.Element("COL")
tableNode.append(colNode)
colNode.set('x', "{:.2f}".format(prevCut))
colNode.set('width', "{:.2f}".format(cut - prevCut))
colNode.set('y', str(table.getY()))
colNode.set('height', str(table.getHeight()))
colNode.set('id', str(index))
def createRef(self, doc):
"""
create a ref file from the xml one
"""
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(doc,
listPages=range(self.firstPage,
self.lastPage + 1))
root = etree.Element("DOCUMENT")
refdoc = etree.ElementTree(root)
for page in self.ODoc.getPages():
#imageFilename="..\col\30275\S_Freyung_021_0001.jpg" width="977.52" height="780.0">
pageNode = etree.Element('PAGE')
pageNode.set("number", page.getAttribute('number'))
pageNode.set("pagekey",
os.path.basename(page.getAttribute('imageFilename')))
pageNode.set("width", page.getAttribute('width'))
pageNode.set("height", page.getAttribute('height'))
root.append(pageNode)
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
dCol = {}
tableNode = etree.Element('TABLE')
tableNode.set("x", table.getAttribute('x'))
tableNode.set("y", table.getAttribute('y'))
tableNode.set("width", table.getAttribute('width'))
tableNode.set("height", table.getAttribute('height'))
pageNode.append(tableNode)
for cell in table.getAllNamedObjects(XMLDSTABLECELLClass):
try:
dCol[int(cell.getAttribute("col"))].append(cell)
except KeyError:
dCol[int(cell.getAttribute("col"))] = [cell]
lXcuts = []
for colid in sorted(dCol.keys()):
lXcuts.append(
min(list(map(lambda x: x.getX(), dCol[colid]))))
self.createColumnsWithCuts(lXcuts, table, tableNode)
return refdoc
def createRefPerPage(self, doc):
"""
create a ref file from the xml one
for DAS 2018: one ref per graph(page)
"""
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(doc,
listPages=range(self.firstPage,
self.lastPage + 1))
dRows = {}
for page in self.ODoc.getPages():
#imageFilename="..\col\30275\S_Freyung_021_0001.jpg" width="977.52" height="780.0">
pageNode = etree.Element('PAGE')
# pageNode.set("number",page.getAttribute('number'))
#SINGLER PAGE pnum=1
pageNode.set("number", '1')
pageNode.set("imageFilename", page.getAttribute('imageFilename'))
pageNode.set("width", page.getAttribute('width'))
pageNode.set("height", page.getAttribute('height'))
root = etree.Element("DOCUMENT")
refdoc = etree.ElementTree(root)
root.append(pageNode)
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
tableNode = etree.Element('TABLE')
tableNode.set("x", table.getAttribute('x'))
tableNode.set("y", table.getAttribute('y'))
tableNode.set("width", table.getAttribute('width'))
tableNode.set("height", table.getAttribute('height'))
pageNode.append(tableNode)
for cell in table.getAllNamedObjects(XMLDSTABLECELLClass):
try:
dRows[int(cell.getAttribute("row"))].append(cell)
except KeyError:
dRows[int(cell.getAttribute("row"))] = [cell]
lYcuts = []
for rowid in sorted(dRows.keys()):
# print rowid, min(map(lambda x:x.getY(),dRows[rowid]))
lYcuts.append(
min(list(map(lambda x: x.getY(), dRows[rowid]))))
self.createRowsWithCuts(lYcuts, table, tableNode)
self.outputFileName = os.path.basename(
page.getAttribute('imageFilename')[:-3] + 'ref')
print(self.outputFileName)
self.writeDom(refdoc, bIndent=True)
return refdoc
class TableProcessing(Component.Component):
usage = ""
version = "v.01"
description = "description: table layout analysis based on template"
sCOL = "col"
sMPXMLExtension = ".mpxml"
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "TableProcessing", self.usage,
self.version, self.description)
self.colid = None
self.docid = None
self.bFullCol = False
# generate MPXML using Ext
self.useExtForMPXML = False
self.bRegenerateMPXML = False
self.sRowModelName = None
self.sRowModelDir = None
self.sHTRmodel = None
self.sDictName = None
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
if "coldir" in dParams:
self.coldir = dParams["coldir"]
if "colid" in dParams:
self.colid = dParams["colid"]
if "colid" in dParams:
self.docid = dParams["docid"]
if "useExt" in dParams:
self.useExtForMPXML = dParams["useExt"]
if 'regMPXML' in dParams:
self.bRegenerateMPXML = True
if "rowmodelname" in dParams:
self.sRowModelName = dParams["rowmodelname"]
if "rowmodeldir" in dParams:
self.sRowModelDir = dParams["rowmodeldir"]
if "htrmodel" in dParams:
self.sHTRmodel = dParams["htrmodel"]
if "dictname" in dParams:
self.sDictName = dParams["dictname"]
# Connection to Transkribus
self.myTrKCient = None
self.persist = False
self.loginInfo = False
if dParams.has_key("server"):
self.server = dParams["server"]
if dParams.has_key("persist"):
self.persist = dParams["persist"]
if dParams.has_key("login"):
self.loginInfo = dParams["login"]
def login(self, trnskrbs_client, trace=None, traceln=None):
"""
deal with the complicated login variants...
-trace and traceln are optional print methods
return True or raises an exception
"""
DEBUG = True
bOk = False
if self.persist:
#try getting some persistent session token
if DEBUG and trace:
trace(" ---login--- Try reusing persistent session ... ")
try:
bOk = trnskrbs_client.reusePersistentSession()
if DEBUG and traceln: traceln("OK!")
except:
if DEBUG and traceln: traceln("Failed")
if not bOk:
if self.loginInfo:
login, pwd = self.loginInfo, self.pwd
else:
if trace:
DEBUG and trace(
" ---login--- no login provided, looking for stored credentials... "
)
login, pwd = trnskrbs_client.getStoredCredentials(bAsk=False)
if DEBUG and traceln: traceln("OK")
if DEBUG and traceln:
trace(" ---login--- logging onto Transkribus as %s " % login)
trnskrbs_client.auth_login(login, pwd)
if DEBUG and traceln: traceln("OK")
bOk = True
return bOk
def downloadCollection(self,
colid,
destDir,
docid,
bNoImg=True,
bForce=False):
"""
download colID
replace destDir by '.' ?
"""
destDir = "."
# options.server, proxies, loggingLevel=logging.WARN)
#download
downloader = TranskribusDownloader(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
downloader.setSessionId(self.myTrKCient.getSessionId())
traceln("- Downloading collection %s to folder %s" %
(colid, os.path.abspath(destDir)))
# col_ts, colDir = downloader.downloadCollection(colid, destDir, bForce=options.bForce, bNoImage=options.bNoImage)
col_ts, colDir, ldocids, dFileListPerDoc = downloader.downloadCollection(
colid, destDir, bForce=bForce, bNoImage=bNoImg, sDocId=docid)
traceln("- Done")
with open(os.path.join(colDir, "config.txt"), "w") as fd:
fd.write("server=%s\nforce=%s\nstrict=%s\n" %
(self.server, True, False))
downloader.generateCollectionMultiPageXml(
os.path.join(colDir, TableProcessing.sCOL), dFileListPerDoc, False)
traceln('- Done, see in %s' % colDir)
return ldocids
def upLoadDocument(self,
colid,
coldir,
docid,
sNote="",
sTranscripExt='.mpxml'):
"""
download colID
"""
# options.server, proxies, loggingLevel=logging.WARN)
#download
# uploader = TranskribusTranscriptUploader(self.server,self.proxies)
uploader = TranskribusDUTranscriptUploader(
self.myTrKCient.getServerUrl(), self.myTrKCient.getProxies())
uploader.setSessionId(self.myTrKCient.getSessionId())
traceln("- uploading document %s to collection %s" % (docid, colid))
uploader.uploadDocumentTranscript(colid,
docid,
os.path.join(coldir, sCOL),
sNote,
'NLE Table',
sTranscripExt,
iVerbose=False)
traceln("- Done")
return
def applyLA_URO(self, colid, docid, nbpages):
"""
apply textline finder
"""
# do the job...
# if options.trp_doc:
# trpdoc = json.load(codecs.open(options.trp_doc, "rb",'utf-8'))
# docId,sPageDesc = doer.buildDescription(colId,options.docid,trpdoc)
traceln('process %s pages...' % nbpages)
lretJobIDs = []
for i in range(1, nbpages + 1):
LA = DoLAbatch(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
LA._trpMng.setSessionId(self.myTrKCient.getSessionId())
LA.setSessionId(self.myTrKCient.getSessionId())
_, sPageDesc = LA.buildDescription(colid, "%s/%s" % (docid, i))
sPageDesc = LA.jsonToXMLDescription(sPageDesc)
_, lJobIDs = LA.run(colid, sPageDesc, "CITlabAdvancedLaJob", False)
traceln(lJobIDs)
lretJobIDs.extend(lJobIDs)
traceln("- LA running for page %d job:%s" % (i, lJobIDs))
return lretJobIDs
def applyHTRForRegions(self, colid, docid, nbpages, modelname, dictionary):
"""
apply an htr model at region level
"""
htrComp = DoHtrRnn(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
htrComp._trpMng.setSessionId(self.myTrKCient.getSessionId())
htrComp.setSessionId(self.myTrKCient.getSessionId())
_, sPageDesc = htrComp.buildDescription(colid,
"%s/%s" % (docid, nbpages))
sPages = "1-%d" % (nbpages)
sModelID = None
# get modelID
lColModels = self.myTrKCient.listRnns(colid)
for model in lColModels:
# print model['htrId'], type(model['htrId']), modelname,type(modelname)
if str(model['htrId']) == str(modelname):
sModelID = model['htrId']
traceln('model id = %s' % sModelID)
#some old? models do not have params field
# try: traceln("%s\t%s\t%s" % (model['htrId'],model['name'],model['params']))
# except KeyError: traceln("%s\t%s\tno params" % (model['htrId'],model['name']))
if sModelID == None:
raise Exception, "no model ID found for %s" % (modelname)
ret = htrComp.htrRnnDecode(colid,
sModelID,
dictionary,
docid,
sPageDesc,
bDictTemp=False)
traceln(ret)
return ret
def applyHTR(self, colid, docid, nbpages, modelname, dictionary):
"""
apply HTR on docid
htr id is needed: we have htrmodename
"""
htrComp = DoHtrRnn(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
htrComp._trpMng.setSessionId(self.myTrKCient.getSessionId())
htrComp.setSessionId(self.myTrKCient.getSessionId())
_, sPageDesc = htrComp.buildDescription(colid,
"%s/%s" % (docid, nbpages))
sPages = "1-%d" % (nbpages)
sModelID = None
# get modelID
lColModels = self.myTrKCient.listRnns(colid)
for model in lColModels:
# print model['htrId'], type(model['htrId']), modelname,type(modelname)
if str(model['htrId']) == str(modelname):
sModelID = model['htrId']
traceln('model id = %s' % sModelID)
#some old? models do not have params field
# try: traceln("%s\t%s\t%s" % (model['htrId'],model['name'],model['params']))
# except KeyError: traceln("%s\t%s\tno params" % (model['htrId'],model['name']))
if sModelID == None:
raise Exception, "no model ID found for %s" % (modelname)
ret = htrComp.htrRnnDecode(colid,
sModelID,
dictionary,
docid,
sPageDesc,
bDictTemp=False)
traceln(ret)
return ret
def extractFileNamesFromMPXML(self, mpxmldoc):
"""
to insure correct file order !
duplicated form performCVLLA.py
"""
xmlpath = os.path.abspath(os.path.join(self.coldir, sCOL, self.docid))
lNd = PageXml.getChildByName(mpxmldoc.getRootElement(), 'Page')
# for i in lNd:print i
return map(
lambda x: "%s%s%s.xml" %
(xmlpath, os.sep, x.prop('imageFilename')[:-4]), lNd)
def processDocument(self, coldir, colid, docid, dom=None):
"""
process a single document
1 python ../../src/xml_formats/PageXml.py trnskrbs_5400/col/17442 --ext=pxml
2 python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form
3 python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400
4 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
5 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml
6 python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442
7 python ../../../TranskribusPyClient/src/TranskribusCommands/do_htrRnn.py <model-name> <dictionary-name> 5400 17442
wait
8 python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force
#covnert to ds
9 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
10 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml --doie --usetemplate
"""
#create Transkribus client
self.myTrKCient = TranskribusClient(sServerUrl=self.server,
proxies={},
loggingLevel=logging.WARN)
#login
_ = self.login(self.myTrKCient, trace=trace, traceln=traceln)
# self.downloadCollection(colid,coldir,docid,bNoImg=False,bForce=True)
## load dom
if dom is None:
self.inputFileName = os.path.abspath(
os.path.join(coldir, TableProcessing.sCOL,
docid + TableProcessing.sMPXMLExtension))
mpxml_doc = self.loadDom()
nbPages = MultiPageXml.getNBPages(mpxml_doc)
else:
# load provided mpxml
mpxml_doc = dom
nbPages = MultiPageXml.getNBPages(mpxml_doc)
# ### table registration: need to compute/select??? the template
# # perform LA separator, table registration, baseline with normalization
# #python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form
# tableregtool= LAProcessor()
# # latool.setParams(dParams)
# tableregtool.coldir = coldir
# tableregtool.docid = docid
# tableregtool.bTemplate, tableregtool.bSeparator , tableregtool.bBaseLine , tableregtool.bRegularTextLine = True,False,False,False
# # creates xml and a new mpxml
# mpxml_doc,nbPages = tableregtool.performLA(mpxml_doc)
#
#
# self.upLoadDocument(colid, coldir,docid,sNote='NLE workflow;table reg done')
lJobIDs = self.applyLA_URO(colid, docid, nbPages)
return
bWait = True
assert lJobIDs != []
jobid = lJobIDs[-1]
traceln("waiting for job %s" % jobid)
while bWait:
dInfo = self.myTrKCient.getJobStatus(jobid)
bWait = dInfo['state'] not in ['FINISHED', 'FAILED']
## coldir???
self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)
##STOP HERE FOR DAS newx testset:
return
# tag text for BIES cell
#python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400
"""
needed : doer = DU_ABPTable_TypedCRF(sModelName, sModelDir,
"""
doer = DU_ABPTable_TypedCRF(self.sRowModelName, self.sRowModelDir)
doer.load()
## needed predict at file level, and do not store dom, but return it
rowpath = os.path.join(coldir, "col")
BIESFiles = doer.predict([rowpath], docid)
BIESDom = self.loadDom(BIESFiles[0])
# res= BIESDom.saveFormatFileEnc('test.mpxml', "UTF-8",True)
# MPXML2DS
#python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
dsconv = primaAnalysis()
DSBIESdoc = dsconv.convert2DS(BIESDom, self.docid)
# create XMLDOC object
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(
DSBIESdoc) #,listPages = range(self.firstPage,self.lastPage+1))
# create row
#python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml
rdc = RowDetection()
rdc.findRowsInDoc(self.ODoc)
#python ../../src/xml_formats/DS2PageXml.py -i trnskrbs_5400/out/17442.ds_xml --multi
# DS2MPXML
DS2MPXML = DS2PageXMLConvertor()
lPageXml = DS2MPXML.run(self.ODoc.getDom())
if lPageXml != []:
# if DS2MPXML.bMultiPages:
newDoc = MultiPageXml.makeMultiPageXmlMemory(
map(lambda xy: xy[0], lPageXml))
outputFileName = os.path.join(
self.coldir, sCOL,
self.docid + TableProcessing.sMPXMLExtension)
newDoc.write(outputFileName,
xml_declaration=True,
encoding="UTF-8",
pretty_print=True)
# else:
# DS2MPXML.storePageXmlSetofFiles(lPageXml)
return
#upload
# python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442
self.upLoadDocument(colid,
coldir,
docid,
sNote='NLE workflow;table row done')
## apply HTR
## how to deal with specific dictionaries?
## here need to know the ontology and the template
nbPages = 1
jobid = self.applyHTR(colid, docid, nbPages, self.sHTRmodel,
self.sDictName)
bWait = True
traceln("waiting for job %s" % jobid)
while bWait:
dInfo = self.myTrKCient.getJobStatus(jobid)
bWait = dInfo['state'] not in ['FINISHED', 'FAILED', 'CANCELED']
# download where???
# python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force
# coldir is not right!! coldir must refer to the parent folder!
self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)
#done!!
# IE extr
## not here: specific to a usecas
#python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml --doie --usetemplate
def processCollection(self, coldir):
"""
process all files in a colelction
need mpxml files
"""
lsDocFilename = sorted(
glob.iglob(
os.path.join(coldir, "*" + TableProcessing.sMPXMLExtension)))
lDocId = []
for sDocFilename in lsDocFilename:
sDocId = os.path.basename(
sDocFilename)[:-len(TableProcessing.sMPXMLExtension)]
try:
docid = int(sDocId)
lDocId.append(docid)
except ValueError:
traceln("Warning: folder %s : %s invalid docid, IGNORING IT" %
(self.coldir, sDocId))
continue
# process each document
for docid in lDocId:
traceln("Processing %s : %s " % (self.coldir, sDocId))
self.processDocument(self.colid, docid)
traceln("\tProcessing done for %s " % (self.coldir, sDocId))
def processParameters(self):
"""
what to do with the parameters provided by the command line
"""
if self.colid is None:
print('collection id missing!')
sys.exit(1)
self.bFullCol = self.docid != None
if self.bRegenerateMPXML and self.docid is not None:
l = glob.glob(os.path.join(self.coldir, sCOL, self.docid,
"*.pxml"))
doc = MultiPageXml.makeMultiPageXml(l)
outputFileName = os.path.join(
self.coldir, sCOL,
self.docid + TableProcessing.sMPXMLExtension)
doc.write(outputFileName,
xml_declaration=True,
encoding="UTF-8",
pretty_print=True)
return doc
return None
def run(self):
"""
process at collection level or document level
"""
newMPXML = self.processParameters()
if self.bFullCol is None:
self.processCollection(self.colid)
else:
self.processDocument(self.coldir, self.colid, self.docid, newMPXML)
class TableProcessing(Component.Component):
usage = ""
version = "v.01"
description = "description: table layout analysis based on template"
sCOL = "col"
sMPXMLExtension = ".mpxml"
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "TableProcessing", self.usage,
self.version, self.description)
self.colid = None
self.docid = None
self.bFullCol = False
# generate MPXML using Ext
self.useExtForMPXML = False
self.bRegenerateMPXML = False
self.sRowModelName = None
self.sRowModelDir = None
self.sHTRmodel = None
self.sDictName = None
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
if "coldir" in dParams:
self.coldir = dParams["coldir"]
if "colid" in dParams:
self.colid = dParams["colid"]
if "colid" in dParams:
self.docid = dParams["docid"]
if "useExt" in dParams:
self.useExtForMPXML = dParams["useExt"]
if 'mergeTLC' in dParams:
self.bUROCVLMerge = dParams["mergeTLC"]
if 'regMPXML' in dParams:
self.bRegenerateMPXML = dParams["regMPXML"]
if "rowmodelname" in dParams:
self.sRowModelName = dParams["rowmodelname"]
if "rowmodeldir" in dParams:
self.sRowModelDir = dParams["rowmodeldir"]
if "htrmodel" in dParams:
self.sHTRmodel = dParams["htrmodel"]
if "dictname" in dParams:
self.sDictName = dParams["dictname"]
# Connection to Transkribus
self.myTrKCient = None
self.persist = False
self.loginInfo = False
if "server" in dParams:
self.server = dParams["server"]
if "persist" in dParams:
self.persist = dParams["persist"]
if "login" in dParams:
self.loginInfo = dParams["login"]
def login(self, trnskrbs_client, trace=None, traceln=None):
"""
deal with the complicated login variants...
-trace and traceln are optional print methods
return True or raises an exception
"""
DEBUG = True
bOk = False
if self.persist:
#try getting some persistent session token
if DEBUG and trace:
trace(" ---login--- Try reusing persistent session ... ")
try:
bOk = trnskrbs_client.reusePersistentSession()
if DEBUG and traceln: traceln("OK!")
except:
if DEBUG and traceln: traceln("Failed")
if not bOk:
if self.loginInfo:
login, pwd = self.loginInfo, self.pwd
else:
if trace:
DEBUG and trace(
" ---login--- no login provided, looking for stored credentials... "
)
login, pwd = trnskrbs_client.getStoredCredentials(bAsk=False)
if DEBUG and traceln: traceln("OK")
if DEBUG and traceln:
trace(" ---login--- logging onto Transkribus as %s " % login)
trnskrbs_client.auth_login(login, pwd)
if DEBUG and traceln: traceln("OK")
bOk = True
return bOk
def downloadCollection(self,
colid,
destDir,
docid,
bNoImg=True,
bForce=False):
"""
download colID
replace destDir by '.' ?
"""
destDir = "."
# options.server, proxies, loggingLevel=logging.WARN)
#download
downloader = TranskribusDownloader(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
downloader.setSessionId(self.myTrKCient.getSessionId())
traceln("- Downloading collection %s to folder %s" %
(colid, os.path.abspath(destDir)))
# col_ts, colDir = downloader.downloadCollection(colid, destDir, bForce=options.bForce, bNoImage=options.bNoImage)
col_ts, colDir, ldocids, dFileListPerDoc = downloader.downloadCollection(
colid, destDir, bForce=bForce, bNoImage=bNoImg, sDocId=docid)
traceln("- Done")
with open(os.path.join(colDir, "config.txt"), "w") as fd:
fd.write("server=%s\nforce=%s\nstrict=%s\n" %
(self.server, True, False))
downloader.generateCollectionMultiPageXml(
os.path.join(colDir, TableProcessing.sCOL), dFileListPerDoc, False)
traceln('- Done, see in %s' % colDir)
return ldocids
def upLoadDocument(self,
colid,
coldir,
docid,
sNote="",
sTranscripExt='.mpxml'):
"""
download colID
"""
# options.server, proxies, loggingLevel=logging.WARN)
#download
# uploader = TranskribusTranscriptUploader(self.server,self.proxies)
uploader = TranskribusDUTranscriptUploader(
self.myTrKCient.getServerUrl(), self.myTrKCient.getProxies())
uploader.setSessionId(self.myTrKCient.getSessionId())
traceln("- uploading document %s to collection %s" % (docid, colid))
uploader.uploadDocumentTranscript(colid,
docid,
os.path.join(coldir, sCOL),
sNote,
'NLE Table',
sTranscripExt,
iVerbose=False)
traceln("- Done")
return
def applyLA_URO(self, colid, docid, nbpages):
"""
apply textline finder
"""
# do the job...
# if options.trp_doc:
# trpdoc = json.load(codecs.open(options.trp_doc, "rb",'utf-8'))
# docId,sPageDesc = doer.buildDescription(colId,options.docid,trpdoc)
traceln('process %s pages...' % nbpages)
lretJobIDs = []
for i in range(1, nbpages + 1):
LA = DoLAbatch(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
LA._trpMng.setSessionId(self.myTrKCient.getSessionId())
LA.setSessionId(self.myTrKCient.getSessionId())
_, sPageDesc = LA.buildDescription(colid, "%s/%s" % (docid, i))
sPageDesc = LA.jsonToXMLDescription(sPageDesc)
_, lJobIDs = LA.run(colid, sPageDesc, "CITlabAdvancedLaJob", False)
traceln(lJobIDs)
lretJobIDs.extend(lJobIDs)
traceln("- LA running for page %d job:%s" % (i, lJobIDs))
return lretJobIDs
def applyHTRForRegions(self, colid, docid, nbpages, modelname, dictionary):
"""
apply an htr model at region level
"""
htrComp = DoHtrRnn(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
htrComp._trpMng.setSessionId(self.myTrKCient.getSessionId())
htrComp.setSessionId(self.myTrKCient.getSessionId())
_, sPageDesc = htrComp.buildDescription(colid,
"%s/%s" % (docid, nbpages))
sPages = "1-%d" % (nbpages)
sModelID = None
# get modelID
lColModels = self.myTrKCient.listRnns(colid)
for model in lColModels:
# print model['htrId'], type(model['htrId']), modelname,type(modelname)
if str(model['htrId']) == str(modelname):
sModelID = model['htrId']
traceln('model id = %s' % sModelID)
#some old? models do not have params field
# try: traceln("%s\t%s\t%s" % (model['htrId'],model['name'],model['params']))
# except KeyError: traceln("%s\t%s\tno params" % (model['htrId'],model['name']))
if sModelID == None:
raise Exception("no model ID found for %s" % (modelname))
ret = htrComp.htrRnnDecode(colid,
sModelID,
dictionary,
docid,
sPageDesc,
bDictTemp=False)
traceln(ret)
return ret
def applyHTR(self, colid, docid, nbpages, modelname, dictionary):
"""
apply HTR on docid
htr id is needed: we have htrmodename
"""
htrComp = DoHtrRnn(self.myTrKCient.getServerUrl(),
self.myTrKCient.getProxies())
htrComp._trpMng.setSessionId(self.myTrKCient.getSessionId())
htrComp.setSessionId(self.myTrKCient.getSessionId())
_, sPageDesc = htrComp.buildDescription(colid,
"%s/%s" % (docid, nbpages))
sPages = "1-%d" % (nbpages)
sModelID = None
# get modelID
lColModels = self.myTrKCient.listRnns(colid)
for model in lColModels:
# print model['htrId'], type(model['htrId']), modelname,type(modelname)
if str(model['htrId']) == str(modelname):
sModelID = model['htrId']
traceln('model id = %s' % sModelID)
#some old? models do not have params field
# try: traceln("%s\t%s\t%s" % (model['htrId'],model['name'],model['params']))
# except KeyError: traceln("%s\t%s\tno params" % (model['htrId'],model['name']))
if sModelID == None:
raise Exception("no model ID found for %s" % (modelname))
ret = htrComp.htrRnnDecode(colid,
sModelID,
dictionary,
docid,
sPageDesc,
bDictTemp=False)
traceln(ret)
return ret
def overlapX(self, zoneA, zoneB):
[x11, y11, x12, y12] = zoneA.getBoundingBox(
) #self.getX(),self.getY(),self.getHeight(),self.getWidth()
[x21, y21, x22, y22] = zoneB.getBoundingBox()
[a1, a2] = x11, x12
[b1, b2] = x21, x22 #zoneB.getX(),zoneB.getX()+ zoneB.getWidth()
return min(a2, b2) >= max(a1, b1)
def overlapY(self, zoneA, zoneB):
[x11, y11, x12, y12] = zoneA.getBoundingBox(
) #self.getX(),self.getY(),self.getHeight(),self.getWidth()
[x21, y21, x22, y22] = zoneB.getBoundingBox()
[a1, a2] = y11, y12
[b1, b2] = y22, y22 #zone.getY(),zone.getY() + zone.getHeight()
return min(a2, b2) >= max(a1, b1)
def signedOverlap(self, zoneA, zoneB):
"""
overlap self and zone
return surface of self in zone
"""
[x11, y11, x12, y12] = zoneA.getBoundingBox(
) #self.getX(),self.getY(),self.getHeight(),self.getWidth()
[x21, y21, x22, y22] = zoneB.getBoundingBox(
) #.getX(),zone.getY(),zone.getHeight(),zone.getWidth()
w1 = x12 - x11
h1 = y12 - y11
fOverlap = 0.0
if self.overlapX(zoneA, zoneB) and self.overlapY(zoneA, zoneB):
s1 = w1 * h1
# possible ?
if s1 == 0: s1 = 1.0
#intersection
nx1 = max(x11, x21)
nx2 = min(x12, x22)
ny1 = max(y11, y21)
ny2 = min(y12, y22)
h = abs(nx2 - nx1)
w = abs(ny2 - ny1)
inter = h * w
if inter > 0:
fOverlap = inter / s1
else:
# if overX and Y this is not possible !
fOverlap = 0.0
return fOverlap
def mergeBaselineCells(self, coldir, colid, docid):
"""
Take a file (pxml) with stuff processed on Transkribus
Tale the CVL template tool xml (xml)
merge them
regenerate a mpxml
"""
xmlpath = os.path.abspath(os.path.join(coldir, sCOL, docid))
# print (xmlpath)
mpxml = xmlpath + ".mpxml"
mpxmldoc = etree.parse(mpxml)
lxml = glob.glob(os.path.join(xmlpath, "*.xml"))
pxmldoc = MultiPageXml.makeMultiPageXml(lxml)
lhtrxml = glob.glob(os.path.join(xmlpath, "*.pxml"))
mpxmldoc = MultiPageXml.makeMultiPageXml(lhtrxml)
lPXMLPage = PageXml.getChildByName(mpxmldoc.getroot(), 'Page')
lXMLPage = PageXml.getChildByName(pxmldoc.getroot(), 'Page')
assert len(lXMLPage) == len(lPXMLPage)
for i, cvlpage in enumerate(lXMLPage):
## remove TextRegion from xcvlpage
lTextRegions = PageXml.getChildByName(cvlpage, 'TextRegion')
for tr in lTextRegions:
tr.getparent().remove(tr)
pxmlpage = lPXMLPage[i]
lTL = []
lTextRegions = PageXml.getChildByName(pxmlpage, 'TextRegion')
for x in lTextRegions:
lTL.extend(PageXml.getChildByName(x, 'TextLine'))
ltable = PageXml.getChildByName(cvlpage, 'TableRegion')
if len(ltable) == 0:
raise "NO TABLE"
lCells = PageXml.getChildByName(ltable[0], 'TableCell')
lC = [Polygon(PageXml.getPointList(c)) for c in lCells]
lT = [Polygon(PageXml.getPointList(t)) for t in lTL]
for i, tl in enumerate(lT):
## normalization
lCoordsPoints = PageXml.getChildByName(lTL[i], 'Coords')
lCoordsB = PageXml.getChildByName(lTL[i], 'Baseline')
coordB = lCoordsB[0]
coord = lCoordsPoints[0]
iHeight = 30 # in pixel
x1, y1, x2, y2 = Polygon(
PageXml.getPointList(coordB)).getBoundingBox()
if coord is not None:
coord.set(
'points', "%d,%d %d,%d %d,%d %d,%d" %
(x1, y1 - iHeight, x2, y1 - iHeight, x2, y2, x1, y2))
tl = Polygon(PageXml.getPointList(coordB))
lOverlap = []
for _, c in enumerate(lC):
# print (lCells[j].get('row'),lCells[j].get('col'), self.signedOverlap(c,tl),tl.getBoundingBox(),c.getBoundingBox())
lOverlap.append(self.signedOverlap(
c, tl)) #.getBoundingBox()))
## region of the same size as the textline
# print (j,max(lOverlap),lOverlap.index(max(lOverlap)))
if max(lOverlap) == 0:
region = PageXml.createPageXmlNode('TextRegion')
cvlpage.append(region)
region.append(lTL[i])
else:
cell = lCells[lOverlap.index(max(lOverlap))]
cell.append(lTL[i])
# print (cell.get('row'),cell.get('col'),''.join(lTL[i].itertext()))
pxmldoc.write(mpxml)
"""
lOverlap=[]
for region in lRegions:
lOverlap.append(self.signedRatioOverlap(region))
if max(lOverlap) == 0: return None
return lRegions[lOverlap.index(max(lOverlap))]
"""
"""
fOverlap = 0.0
if self.overlapX(zone) and self.overlapY(zone):
[x11,y11,x12,y12] = [x1,y1,x1+w1,y1+h1]
[x21,y21,x22,y22] = [x2,y2,x2+w2,y2+h2]
s1 = w1 * h1
# possible ?
if s1 == 0: s1 = 1.0
#intersection
nx1 = max(x11,x21)
nx2 = min(x12,x22)
ny1 = max(y11,y21)
ny2 = min(y12,y22)
h = abs(nx2 - nx1)
w = abs(ny2 - ny1)
inter = h * w
if inter > 0 :
fOverlap = inter/s1
else:
# if overX and Y this is not possible !
fOverlap = 0.0
return fOverlap
"""
def extractFileNamesFromMPXML(self, mpxmldoc):
"""
to insure correct file order !
duplicated form performCVLLA.py
"""
xmlpath = os.path.abspath(os.path.join(self.coldir, sCOL, self.docid))
lNd = PageXml.getChildByName(mpxmldoc.getRootElement(), 'Page')
# for i in lNd:print i
return map(
lambda x: "%s%s%s.xml" %
(xmlpath, os.sep, x.prop('imageFilename')[:-4]), lNd)
def processDocument(self, coldir, colid, docid, dom=None):
"""
process a single document
1 python ../../src/xml_formats/PageXml.py trnskrbs_5400/col/17442 --ext=pxml
2 python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form
3 python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400
4 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
5 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml
6 python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442
7 python ../../../TranskribusPyClient/src/TranskribusCommands/do_htrRnn.py <model-name> <dictionary-name> 5400 17442
wait
8 python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force
#covnert to ds
9 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
10 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml --doie --usetemplate
"""
#create Transkribus client
self.myTrKCient = TranskribusClient(sServerUrl=self.server,
proxies={},
loggingLevel=logging.WARN)
#login
_ = self.login(self.myTrKCient, trace=trace, traceln=traceln)
# self.downloadCollection(colid,coldir,docid,bNoImg=False,bForce=True)
## load dom
if dom is None:
self.inputFileName = os.path.abspath(
os.path.join(coldir, TableProcessing.sCOL,
docid + TableProcessing.sMPXMLExtension))
mpxml_doc = self.loadDom()
nbPages = MultiPageXml.getNBPages(mpxml_doc)
else:
# load provided mpxml
mpxml_doc = dom
nbPages = MultiPageXml.getNBPages(mpxml_doc)
# ### table registration: need to compute/select??? the template
# # perform LA separator, table registration, baseline with normalization
# #python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form
# tableregtool= LAProcessor()
# # latool.setParams(dParams)
# tableregtool.coldir = coldir
# tableregtool.docid = docid
# tableregtool.bTemplate, tableregtool.bSeparator , tableregtool.bBaseLine , tableregtool.bRegularTextLine = True,False,False,False
# # creates xml and a new mpxml
# mpxml_doc,nbPages = tableregtool.performLA(mpxml_doc)
#
#
# self.upLoadDocument(colid, coldir,docid,sNote='NLE workflow;table reg done')
lJobIDs = self.apply_URO(colid, docid, nbPages)
return
bWait = True
assert lJobIDs != []
jobid = lJobIDs[-1]
traceln("waiting for job %s" % jobid)
while bWait:
dInfo = self.myTrKCient.getJobStatus(jobid)
bWait = dInfo['state'] not in ['FINISHED', 'FAILED']
## coldir???
self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)
##STOP HERE FOR DAS newx testset:
return
# tag text for BIES cell
#python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400
"""
needed : doer = DU_ABPTable_TypedCRF(sModelName, sModelDir,
"""
doer = DU_ABPTable_TypedCRF(self.sRowModelName, self.sRowModelDir)
doer.load()
## needed predict at file level, and do not store dom, but return it
rowpath = os.path.join(coldir, "col")
BIESFiles = doer.predict([rowpath], docid)
BIESDom = self.loadDom(BIESFiles[0])
# res= BIESDom.saveFormatFileEnc('test.mpxml', "UTF-8",True)
# MPXML2DS
#python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442
dsconv = primaAnalysis()
DSBIESdoc = dsconv.convert2DS(BIESDom, self.docid)
# create XMLDOC object
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(
DSBIESdoc) #,listPages = range(self.firstPage,self.lastPage+1))
# create row
#python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml
rdc = RowDetection()
rdc.findRowsInDoc(self.ODoc)
#python ../../src/xml_formats/DS2PageXml.py -i trnskrbs_5400/out/17442.ds_xml --multi
# DS2MPXML
DS2MPXML = DS2PageXMLConvertor()
lPageXml = DS2MPXML.run(self.ODoc.getDom())
if lPageXml != []:
# if DS2MPXML.bMultiPages:
newDoc = MultiPageXml.makeMultiPageXmlMemory(
map(lambda xy: xy[0], lPageXml))
outputFileName = os.path.join(
self.coldir, sCOL,
self.docid + TableProcessing.sMPXMLExtension)
newDoc.write(outputFileName,
xml_declaration=True,
encoding="UTF-8",
pretty_print=True)
# else:
# DS2MPXML.storePageXmlSetofFiles(lPageXml)
return
#upload
# python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442
self.upLoadDocument(colid,
coldir,
docid,
sNote='NLE workflow;table row done')
## apply HTR
## how to deal with specific dictionaries?
## here need to know the ontology and the template
## OPTION: put it after LA on server (just one download needed )
nbPages = 1
jobid = self.applyHTR(colid, docid, nbPages, self.sHTRmodel,
self.sDictName)
bWait = True
traceln("waiting for job %s" % jobid)
while bWait:
dInfo = self.myTrKCient.getJobStatus(jobid)
bWait = dInfo['state'] not in ['FINISHED', 'FAILED', 'CANCELED']
# download where???
# python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force
# coldir is not right!! coldir must refer to the parent folder!
self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)
#done!!
# IE extr
## not here: specific to a usecas
#python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml --doie --usetemplate
def processCollection(self, coldir):
"""
process all files in a colelction
need mpxml files
"""
lsDocFilename = sorted(
glob.iglob(
os.path.join(coldir, "*" + TableProcessing.sMPXMLExtension)))
lDocId = []
for sDocFilename in lsDocFilename:
sDocId = os.path.basename(
sDocFilename)[:-len(TableProcessing.sMPXMLExtension)]
try:
docid = int(sDocId)
lDocId.append(docid)
except ValueError:
traceln("Warning: folder %s : %s invalid docid, IGNORING IT" %
(self.coldir, sDocId))
continue
# process each document
for docid in lDocId:
traceln("Processing %s : %s " % (self.coldir, sDocId))
self.processDocument(self.colid, docid)
traceln("\tProcessing done for %s " % (self.coldir, sDocId))
def processParameters(self):
"""
what to do with the parameters provided by the command line
"""
if self.colid is None:
print('collection id missing!')
sys.exit(1)
self.bFullCol = self.docid != None
if self.bRegenerateMPXML and self.docid is not None:
l = glob.glob(os.path.join(self.coldir, sCOL, self.docid,
"*.pxml"))
doc = MultiPageXml.makeMultiPageXml(l)
outputFileName = os.path.join(
self.coldir, sCOL,
self.docid + TableProcessing.sMPXMLExtension)
doc.write(outputFileName,
xml_declaration=True,
encoding="UTF-8",
pretty_print=True)
return doc
return None
def run(self):
"""
process at collection level or document level
"""
newMPXML = self.processParameters()
if self.bFullCol is None:
self.processCollection(self.colid)
else:
if self.bUROCVLMerge:
self.mergeBaselineCells(self.coldir, self.colid, self.docid)
return
self.processDocument(self.coldir, self.colid, self.docid, newMPXML)
class IETest(Component.Component):
"""
"""
usage = ""
version = "v.01"
description = "description: Information Extraction Tool for the ABP collection (READ project)"
#--- INIT -------------------------------------------------------------------------------------------------------------
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "ABP_IE", self.usage, self.version,
self.description)
self.usage = self.usage = "python %prog" + self.usageComponent
self.colname = None
self.docid = None
self.sTemplate = None
self.BuseStoredTemplate = False
# HTR model id
self.htrModelID = None
# IE model
self.sModelDir = None
self.sModelName = None
self.lcsTH = 75
self.page2DS = False
# for --test
self.evalData = None
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
if "coldir" in dParams:
self.colname = dParams["coldir"]
if "docid" in dParams:
self.docid = dParams["docid"]
if "htrid" in dParams:
self.htrModelID = dParams["htrid"]
if "template" in dParams:
self.sTemplate = dParams["template"]
if "UseStoredTemplate" in dParams:
self.BuseStoredTemplate = dParams["UseStoredTemplate"]
if 'modelName' in dParams:
self.sModelName = dParams['modelName']
if 'modelDir' in dParams:
self.sModelDir = dParams['modelDir']
if "2DS" in dParams:
self.page2DS = dParams['2DS']
if "LCSTH" in dParams:
self.lcsTH = dParams['LCSTH']
def labelTable(self, table):
"""
toy example
label columns with tags
"""
table.getColumns()[0].label()
def findNameColumn(self, table):
"""
find the column which corresponds to the people names c
"""
self.bDebug = False
#tag fields with template
lColPos = {}
lColInvName = {}
for cell in table.getCells():
try:
lColPos[cell.getIndex()[1]]
except:
lColPos[cell.getIndex()[1]] = []
if cell.getIndex()[1] < 5:
for field in cell.getFields():
if field is not None:
res = field.applyTaggers(cell)
# res [ (token,label,score) ...]
extractedValues = field.extractLabel(res)
if extractedValues != []:
# extractedValues = map(lambda offset,value,label,score:(value,score),extractedValues)
extractedValues = list(
map(lambda x: (x[1], x[3]), extractedValues))
field.setOffset(res[0])
field.setValue(extractedValues)
# field.addValue(extractedValues)
lColPos[cell.getIndex()[1]].append(field.getName())
try:
lColInvName[field.getName()].append(
cell.getIndex()[1])
except:
lColInvName[field.getName()] = [
cell.getIndex()[1]
]
if self.bDebug:
print('foundXX:', field.getName(),
field.getValue())
cell.resetField()
return max(lColInvName['firstname'],
key=lColInvName['firstname'].count)
def extractData(self, table, myRecord, lTemplate):
"""
layout
tag content
[use scoping for propagating
scoping: for tagging and for data
scope fieldname scope (fiedlname, fieldvalue)]
find if possible a contiguous repetition of records
find layout level for record completion
extract data/record
-inference if IEOnto
"""
# self.bDebug = False
# table.buildNDARRAY()
if lTemplate is not None:
# convert string to tableTemplateObject
template = tableTemplateClass()
template.buildFromPattern(lTemplate)
template.labelTable(table)
else:
return None
# firstNameColIndex =self.findNameColumn(table)
# create a batch for the full page
#tag fields with template
for cell in table.getCells():
if cell.getFields() != []:
if self.bDebug:
print(table.getPage(), cell.getIndex(), cell.getFields(),
cell.getContent())
for field in cell.getFields():
if field is not None:
res = field.applyTaggers(cell)
# res [ (token,label,score) ...]
extractedValues = field.extractLabel(res)
if extractedValues != []:
# extractedValues = map(lambda offset,value,label,score:(value,score),extractedValues)
extractedValues = list(
map(lambda x: (x[1], x[3]), extractedValues))
field.setOffset(res[0])
field.setValue(extractedValues)
# field.addValue(extractedValues)
if self.bDebug:
print('found:', field, field.getValue())
### now at record level ?
### scope = propagation using only docObject (hardcoded ?)
### where to put the propagation mechanism?
# myRecord.propagate(table)
## 'backpropagation: select the rows, and collection subobjects with fields (cells)
for row in table.getRows():
#if not row.isHeaders():
myRecord.addCandidate(row)
# #for each cell: take the record and
# ### FR NOW: TAKE THE FIRST COLUMN
# firstCol = table.getColumns()[0]
# for cell in firstCol.getCells():
# myRecord.addCandidate(cell)
myRecord.rankCandidates()
lcand = myRecord.getCandidates()
# print lcand
# myRecord.display()
def mergeLineAndCells(self, lPages):
"""
assign lines(TEXT) to cells
"""
for page in lPages:
lLines = page.getAllNamedObjects(XMLDSTEXTClass)
lCells = page.getAllNamedObjects(XMLDSTABLECELLClass)
dAssign = {}
for line in lLines:
bestscore = 0.0
for cell in lCells:
ratio = line.ratioOverlap(cell)
if ratio > bestscore:
bestscore = ratio
dAssign[line] = cell
[dAssign[line].addObject(line) for line in dAssign.keys()]
[cell.getObjects().sort(key=lambda x: x.getY()) for cell in lCells]
def testGTText(self, page):
"""
extract region text and parse it
"""
from contentProcessing.taggerTrainKeras import DeepTagger
myTagger = DeepTagger()
myTagger.bPredict = True
myTagger.sModelName = '2mix_cm'
myTagger.dirName = 'IEdata/model/'
myTagger.loadModels()
for region in page.getObjects():
# print region.getContent().encode('utf-8')
res = myTagger.predict([region.getContent()])
try:
res = myTagger.predict([region.getContent()])
# print res
except:
print('SENT WITH ISSUES : [%s]' %
(region.getContent().encode('utf-8')))
def mineTable(self, tabel, dr):
"""
from the current HTR: find the categories in each column (once NER applied)
"""
def selectTemplat(self, lTemplates):
"""
if a list of templates is available:
take a couple pages and perform IE: simply sum up the score of the tagger
"""
def processWithTemplate(self, table, dr):
"""
according to the # of columns, apply the corresponding template
"""
# selection of the dictionaries per columns
# template 5,10: first col = numbering
# find calibration column: abp_names
table.buildNDARRAY()
# print (self.findNameColumn(table))
# lTemplateIE2 = [
# ((slice(1,None),slice(0,1)) ,[ 'numbering'],[ dr.getFieldByName('numbering') ])
# , ((slice(1,None),slice(1,2)) ,[ 'abp_names', 'names_aux','numbering','religion'],[ dr.getFieldByName('lastname'), dr.getFieldByName('firstname'),dr.getFieldByName('religion') ])
# , ((slice(1,None),slice(2,3)) ,[ 'abp_profession','religion' ] ,[ dr.getFieldByName('occupation'), dr.getFieldByName('religion') ])
# , ((slice(1,None),slice(3,4)) ,[ 'abp_location' ] ,[ dr.getFieldByName('location') ])
# , ((slice(1,None),slice(4,5)) ,[ 'abp_family' ] ,[ dr.getFieldByName('situation') ])
# ,((slice(1,None),slice(5,6)) ,[ 'deathreason','artz'] ,[ dr.getFieldByName('deathreason'),dr.getFieldByName('doktor')])
# , ((slice(1,None),slice(6,7)) ,[] , [ ]) #binding
# , ((slice(1,None),slice(7,8)) ,['abp_dates', 'abp_dates' ,'abp_year'] ,[,dr.getFieldByName('deathDate'),dr.getFieldByName('deathYear') ])
# , ((slice(1,None),slice(8,9)) ,[ 'abp_dates','abp_location' ] ,[ dr.getFieldByName('burialDate'),dr.getFieldByName('burialLocation') ])
# , ((slice(1,None),slice(9,10)) ,[ 'abp_age','abp_ageunit'] ,[ dr.getFieldByName('age'), dr.getFieldByName('ageUnit')])
# # , ((slice(1,None),slice(9,10)) ,[ dr.getFieldByName('priester')])
# # , ((slice(1,None),slice(10,11)),[ dr.getFieldByName('notes')])
# ]
#fuzzy
lTemplateIECAL = [
((slice(1, None), slice(0, 4)),
['abp_names', 'names_aux', 'numbering', 'religion'], [
dr.getFieldByName('lastname'),
dr.getFieldByName('firstname'),
dr.getFieldByName('religion')
]),
((slice(1, None), slice(1, 4)), ['abp_profession', 'religion'],
[dr.getFieldByName('occupation'),
dr.getFieldByName('religion')])
]
#detect empty left columns ?
template = tableTemplateClass()
template.buildFromPattern(lTemplateIECAL)
template.labelTable(table)
iRef = self.findNameColumn(table)
lTemplateIE = [
((slice(1, None), slice(iRef, iRef + 1)),
['abp_names', 'names_aux', 'numbering', 'religion'], [
dr.getFieldByName('lastname'),
dr.getFieldByName('firstname'),
dr.getFieldByName('religion')
]),
((slice(1, None), slice(iRef + 1,
iRef + 2)), ['abp_profession', 'religion'],
[dr.getFieldByName('occupation'),
dr.getFieldByName('religion')]),
((slice(1, None), slice(iRef + 2, iRef + 3)), ['abp_location'],
[dr.getFieldByName('location')]),
((slice(1, None), slice(iRef + 3, iRef + 4)), ['abp_family'],
[dr.getFieldByName('situation')])
#[] binding
,
((slice(1, None), slice(iRef + 4,
iRef + 6)), ['abp_deathreason', 'artz'],
[dr.getFieldByName('deathreason'),
dr.getFieldByName('doktor')]),
((slice(1, None), slice(iRef + 5, iRef + 7)),
['abp_dates', 'abp_year'], [
dr.getFieldByName('MonthDayDateGenerator'),
dr.getFieldByName('deathDate'),
dr.getFieldByName('deathYear')
]),
((slice(1, None), slice(iRef + 6, iRef + 8)),
['abp_dates', 'abp_year', 'abp_location'], [
dr.getFieldByName('burialDate'),
dr.getFieldByName('deathYear'),
dr.getFieldByName('burialLocation')
]),
((slice(1, None), slice(iRef + 8,
iRef + 10)), ['abp_age', 'abp_ageunit'],
[dr.getFieldByName('age'),
dr.getFieldByName('ageUnit')])
# , ((slice(1,None),slice(9,10)) ,[ dr.getFieldByName('priester')])
# , ((slice(1,None),slice(10,11)),[ dr.getFieldByName('notes')])
]
# recalibrate template
# # lTemplate = lTemplateIE
# if table.getNbColumns() >= 12:
# lTemplate = lTemplateIE2
# else:
# lTemplate = lTemplateIE
self.extractData(table, dr, lTemplateIE)
# select best solutions
# store inthe proper final format
return dr
def run(self, doc):
"""
main issue: how to select the template: to be done by CVL
assuming IE and htr info are stored in the template
"""
# self.firstPage = 9
# self.lastPage= 9
if self.page2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
dr = deathRecord(self.sModelName, self.sModelDir)
## selection of the templates first with X tables
###
for page in self.lPages:
print("page: ", page.getNumber())
# self.testGTText(page)
# continue
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
if table.getNbRows() < 2:
print("page: %s : not a table? %d/%d" %
(page.getNumber(), table.getNbRows(),
table.getNbColumns()))
continue
if self.BuseStoredTemplate:
# self.processWithTemplate(table, dr)
try:
self.processWithTemplate(table, dr)
except:
print('issue with page %s' % page)
else:
self.mineTable(table, dr)
self.evalData = dr.generateOutput(self.evalData)
# print self.evalData.serialize('utf-8',True)
def generateTestOutput(self):
"""
<PAGE number="1" pagenum="1" nbrecords="10" years="0">
<RECORD lastname="Riesinger" firstname="Korona" role="Verstorbener" location="Neuhofen" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Ringseisen" firstname="Georg" role="Verstorbener" location="Etzing" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Nebel" firstname="Theresia" role="Verstorbener" location="Sandbach" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Schlögl" firstname="Cäcilia" role="Verstorbener" location="Stampfing" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Riedinger" firstname="Theresia" role="Verstorbener" location="Lapperding" occupation="Austragsbäuerin" year="0" month="0" day="0"/>
<RECORD lastname="Wührer" firstname="Joseph" role="Verstorbener" location="Haizing" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Wilmerdinger" firstname="Theresia" role="Verstorbener" location="Hausmanning" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Ratzinger" firstname="Mathias" role="Verstorbener" location="Kafferding" occupation="Bauer" year="0" month="0" day="0"/>
<RECORD lastname="Deixelberger" firstname="Joseph" role="Verstorbener" location="Gaishofen" occupation="Inwohner" year="0" month="0" day="0"/>
<RECORD lastname="Beham" firstname="Martin" role="Verstorbener" location="Socking" occupation="Austragsbauer" year="0" month="0" day="0"/>
</PAGE>
"""
root = etree.Element('DOCUMENT')
self.evalData = etree.ElementTree(root)
for page in lPages:
domp = etree.Elemen('PAGE')
domp.set('number', page.getAttribute('number'))
domp.set('pagenum',
os.path.basename(page.getAttribute('imageFilename'))[:-4])
root.append(domp)
domp.set('template', page.getNode().prop('template'))
domp.set('reftemplate', page.getNode().prop('reftemplate'))
return self.evalData
def testFirstNameLastNameRecord(self, srefData, srunData, bVisual):
"""
test firstname in record
group by page
"""
cntOk = cntErr = cntMissed = 0
# srefData = srefData.decode('utf-8')
#.strip("\n")
RefData = etree.XML(srefData.strip("\n").encode('utf-8'))
RunData = etree.XML(srunData.strip("\n").encode('utf-8'))
lRef = []
lPages = RefData.xpath('//%s' % ('PAGE[@number]'))
lRefKeys = {}
for page in lPages:
pnum = page.get('number')
key = page.get('pagenum')
lRefKeys[key] = 1
xpath = "./%s" % ("RECORD")
lrecord = page.xpath(xpath)
if len(lrecord) == 0:
lRef.append([])
else:
for record in lrecord:
xpath = "./%s" % ("./@firstname")
lf = record.xpath(xpath)
xpath = "./%s" % ("./@lastname")
ln = record.xpath(xpath)
if len(lf) > 0:
lRef.append((pnum, key, lf[0], ln[0]))
lPageMapping = {}
lRun = []
if RunData is not None:
lpages = RunData.xpath('//%s' % ('PAGE[@number]'))
for page in lpages:
pnum = page.get('number')
key = page.get('pagenum')
# key= page.get('number')
if key in lRefKeys.keys():
lPageMapping[key] = pnum
#record level!
xpath = "./%s" % ("RECORD[@firstname and @lastname]")
lrecord = page.xpath(xpath)
if len(lrecord) == 0:
pass
else:
for record in lrecord:
xpath = "./%s" % ("./@firstname")
lf = record.xpath(xpath)
xpath = "./%s" % ("./@lastname")
ln = record.xpath(xpath)
if len(
lf
) > 0: # and lf[0].getContent() != ln[0].getContent():
lRun.append((pnum, key, lf[0], ln[0]))
ltisRefsRunbErrbMiss = list()
for key in lRunPerPage:
# for key in ['Neuoetting_009_05_0150']:
lRun = lRunPerPage[key]
lRef = lRefKeys[key]
runLen = len(lRunPerPage[key])
refLen = len(lRefKeys[key])
bT = False
if refLen <= runLen:
rows = lRef
cols = lRun
else:
rows = lRun
cols = lRef
bT = True
cost_matrix = np.zeros((len(rows), len(cols)), dtype=float)
for a, i in enumerate(rows):
curRef = i
for b, j in enumerate(cols):
runElt = j
ret, val = self.testCompareRecordField(curRef, runElt)
val /= 100
if val == 0:
dist = 10
else:
dist = 1 / val
cost_matrix[a, b] = dist
m = linear_sum_assignment(cost_matrix)
r1, r2 = m
# print (bT,r1,r2)
# print (list(x[2] for x in rows))
# print (list(x[2] for x in cols))
lcsTH = self.lcsTH / 100
lCovered = []
for a, i in enumerate(r2):
# print (key,a,r1[a],i,rows[r1[a]][2],cols[i][2], 1/cost_matrix[r1[a],i])
if 1 / cost_matrix[r1[a], i] > lcsTH:
cntOk += 1
if bT:
ltisRefsRunbErrbMiss.append(
(runElt[1], int(runElt[0]), cols[i], rows[r1[a]],
False, False))
else:
ltisRefsRunbErrbMiss.append(
(runElt[1], int(runElt[0]), rows[r1[a]], cols[i],
False, False))
else:
#too distant: false
if bT:
lCovered.append(i)
ltisRefsRunbErrbMiss.append(
(runElt[1], int(runElt[0]), "", rows[r1[a]], True,
False))
else:
lCovered.append(r1[a])
ltisRefsRunbErrbMiss.append((runElt[1], int(runElt[0]),
"", cols[i], True, False))
cntErr += 1
for iref in r1:
if iref not in r2:
ltisRefsRunbErrbMiss.append((runElt[1], int(runElt[0]),
lRef[iref], '', False, True))
cntMissed += 1
for iref in lCovered:
ltisRefsRunbErrbMiss.append(
(runElt[1], int(runElt[0]), lRef[iref], '', False, True))
cntMissed += 1
ltisRefsRunbErrbMiss.sort(key=lambda x: x[0])
# runLen = len(lRun)
# refLen = len(lRef)
# # bVisual = True
# ltisRefsRunbErrbMiss= list()
# lRefCovered = []
# for i in range(0,len(lRun)):
# iRef = 0
# bFound = False
# bErr , bMiss= False, False
# runElt = lRun[i]
# # print '\t\t===',runElt
# while not bFound and iRef <= refLen - 1:
# curRef = lRef[iRef]
# if runElt and curRef not in lRefCovered and self.testCompareRecordFirstNameLastName(curRef,runElt):
# bFound = True
# lRefCovered.append(curRef)
# iRef+=1
# if bFound:
# if bVisual:print("FOUND:", runElt, ' -- ', lRefCovered[-1])
# cntOk += 1
# else:
# curRef=''
# cntErr += 1
# bErr = True
# if bVisual:print("ERROR:", runElt)
# if bFound or bErr:
# ltisRefsRunbErrbMiss.append( (runElt[1],int(runElt[0]), curRef, runElt,bErr, bMiss) )
# for i,curRef in enumerate(lRef):
# if curRef not in lRefCovered:
# if bVisual:print("MISSED:", curRef)
# ltisRefsRunbErrbMiss.append( (curRef[1],int(lPageMapping[curRef[1]]), curRef, '',False, True) )
# cntMissed+=1
#
# ltisRefsRunbErrbMiss.sort(key=lambda x:x[0])
return (cntOk, cntErr, cntMissed, ltisRefsRunbErrbMiss)
def testRecordField(self, lfieldName, lfieldInRef, srefData, srunData,
bVisual):
"""
test fieldName in record
"""
assert len(lfieldName) == len((lfieldInRef))
for i, f in enumerate(lfieldName):
if lfieldInRef[i] is None: lfieldInRef[i] = f
cntOk = cntErr = cntMissed = 0
# srefData = srefData.decode('utf-8')
#.strip("\n")
RefData = etree.XML(srefData.strip("\n").encode('utf-8'))
RunData = etree.XML(srunData.strip("\n").encode('utf-8'))
lPages = RefData.xpath('//%s' % ('PAGE[@number]'))
lRefKeys = {}
for page in lPages:
pnum = page.get('number')
key = page.get('pagenum')
xpath = "./%s" % ("RECORD")
lrecord = page.xpath(xpath)
if len(lrecord) == 0:
lRef.append([])
else:
for record in lrecord:
lf = []
for fieldInRef in lfieldInRef:
xpath = "./%s" % ("./@%s" % fieldInRef)
ln = record.xpath(xpath)
if ln and len(ln[0]) > 0:
lf.append(ln[0])
if lf != []:
try:
# if (pnum,key,lf) in lRefKeys[key]:
# print ('duplicated',(pnum,key,lf))
# else:
# lRefKeys[key].append((pnum,key,lf))
lRefKeys[key].append((pnum, key, lf))
except KeyError:
lRefKeys[key] = [(pnum, key, lf)]
lRunPerPage = {}
lPageMapping = {}
if RunData:
lpages = RunData.xpath('//%s' % ('PAGE[@number]'))
for page in lpages:
pnum = page.get('number')
key = page.get('pagenum')
lPageMapping[key] = pnum
if key in lRefKeys:
#record level!
xpath = "./%s" % ("RECORD")
lrecord = page.xpath(xpath)
if len(lrecord) == 0:
pass
# lRun.append([])
else:
for record in lrecord:
lf = []
for fieldName in lfieldName:
xpath = "./%s" % ("./@%s" % fieldName)
ln = record.xpath(xpath)
if len(ln) > 0 and len(ln[0]) > 0:
lf.append(ln[0])
if len(lf) == len(lfieldName):
try:
lRunPerPage[key].append((pnum, key, lf))
except KeyError:
lRunPerPage[key] = [(pnum, key, lf)]
ltisRefsRunbErrbMiss = list()
for key in lRunPerPage:
# for key in ['Neuoetting_008_03_0032']:
lRun = lRunPerPage[key]
lRef = lRefKeys[key]
runLen = len(lRunPerPage[key])
refLen = len(lRefKeys[key])
bT = False
if refLen <= runLen:
rows = lRef
cols = lRun
else:
rows = lRun
cols = lRef
bT = True
cost_matrix = np.zeros((len(rows), len(cols)), dtype=float)
for a, i in enumerate(rows):
curRef = i
for b, j in enumerate(cols):
runElt = j
ret, val = self.testCompareRecordField(curRef, runElt)
dist = 100 - val
cost_matrix[a, b] = dist
# print (curRef,runElt,val,dist)
m = linear_sum_assignment(cost_matrix)
r1, r2 = m
if False:
print(len(lRef), lRef)
print(len(lRun), lRun)
print(bT, r1, r2)
lcsTH = self.lcsTH
lCovered = []
lMatched = []
for a, i in enumerate(r2):
# print (key,a,r1[a],i,rows[r1[a]][2],cols[i][2], 100-cost_matrix[r1[a],i])
if 100 - cost_matrix[r1[a, ], i] > lcsTH:
cntOk += 1
if bT:
ltisRefsRunbErrbMiss.append(
(rows[r1[a]][1], int(rows[r1[a]][0]), cols[i][2],
rows[r1[a]][2], False, False))
lMatched.append(i)
else:
ltisRefsRunbErrbMiss.append(
(cols[i][1], int(cols[i][0]), rows[r1[a]][2],
cols[i][2], False, False))
lMatched.append(r1[a])
else:
#too distant: false
if bT:
lCovered.append(i)
ltisRefsRunbErrbMiss.append(
(rows[r1[a]][1], int(rows[r1[a]][0]), "",
rows[r1[a]][2], True, False))
else:
lCovered.append(r1[a])
ltisRefsRunbErrbMiss.append(
(cols[i][1], int(cols[i][0]), "", cols[i][2], True,
False))
cntErr += 1
# print ('matched',lMatched)
for i, iref in enumerate(lRef):
if i not in lMatched:
# print ('not mathced',i,iref)
ltisRefsRunbErrbMiss.append(
(lRef[i][1], int(lPageMapping[lRef[i][1]]), lRef[i][2],
'', False, True))
cntMissed += 1
# else:print('machtg!',i,lRef[i])
ltisRefsRunbErrbMiss.sort(key=lambda x: x[0])
# for x in ltisRefsRunbErrbMiss:
# print (x)
return (cntOk, cntErr, cntMissed, ltisRefsRunbErrbMiss)
def testCompareRecordFirstNameLastName(self,
refdata,
rundata,
bVisual=False):
if refdata[1] != rundata[1]: return False
refall = refdata[2].lower() + refdata[3].lower()
reflen = len(refdata[2]) + len(refdata[3])
runall = rundata[2].lower() + rundata[3].lower()
runlen = len(rundata[2]) + len(rundata[3])
runall.replace('n̄', 'nn')
runall.replace('m̄', 'mm')
return matchLCS(0, (refall, reflen), (runall, runlen))
def testCompareRecordField(self, refdata, rundata, bVisual=False):
# same page !!
if refdata[1] != rundata[1]: return False, 0
if rundata[2] == []: return False, 0
if refdata[2] == []: return False, 0
runall = " ".join(rundata[2]).strip().lower()
refall = " ".join(refdata[2]).strip().lower()
return matchLCS(0, (refall, len(refall)), (runall, len(runall)))
return res, val
def testCompareFullRecord(self, refdata, rundata, bVisual=False):
bOK = True
for i, attr in enumerate(refdata):
bOK = bOK and refdata[i] == rundata[i]
return bOK
################ TEST ##################
def createFakeData(self):
"""
for testing purpose
<PAGE number="1" pagenum="1" nbrecords="10" years="0">
<RECORD lastname="Riesinger" firstname="Korona" role="Verstorbener" location="Neuhofen" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Ringseisen" firstname="Georg" role="Verstorbener" location="Etzing" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Nebel" firstname="Theresia" role="Verstorbener" location="Sandbach" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Schlögl" firstname="Cäcilia" role="Verstorbener" location="Stampfing" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Riedinger" firstname="Theresia" role="Verstorbener" location="Lapperding" occupation="Austragsbäuerin" year="0" month="0" day="0"/>
<RECORD lastname="Wührer" firstname="Joseph" role="Verstorbener" location="Haizing" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Wilmerdinger" firstname="Theresia" role="Verstorbener" location="Hausmanning" occupation="" year="0" month="0" day="0"/>
<RECORD lastname="Ratzinger" firstname="Mathias" role="Verstorbener" location="Kafferding" occupation="Bauer" year="0" month="0" day="0"/>
<RECORD lastname="Deixelberger" firstname="Joseph" role="Verstorbener" location="Gaishofen" occupation="Inwohner" year="0" month="0" day="0"/>
<RECORD lastname="Beham" firstname="Martin" role="Verstorbener" location="Socking" occupation="Austragsbauer" year="0" month="0" day="0"/>
</PAGE>
"""
self.evalData = libxml2.newDoc('1.0')
root = libxml2.newNode('DOCUMENT')
self.evalData.setRootElement(root)
domp = libxml2.newNode('PAGE')
domp.setProp('number', '182')
domp.setProp('nbrecords', 'None')
domp.setProp('years', '1876;1877')
root.addChild(domp)
record = libxml2.newNode('RECORD')
domp.addChild(record)
record.setProp('lastname', 'Riedinger')
record.setProp('firstname', 'Theresia')
print(
etree.tostring(self.evalData,
encoding='unicode',
pretty_print=True))
return self.evalData
def testRun(self, filename, outFile=None):
"""
testRun is responsible for running the component on this file and returning a string that reflects the result in a way
that is understandable to a human and to a program. Nicely serialized Python data or XML is fine
"""
self.evalData = None
doc = self.loadDom(filename)
self.run(doc)
# self.generateTestOutput()
# self.createFakeData()
if outFile: self.writeDom(doc)
# return unicode
return etree.tostring(self.evalData,
encoding='unicode',
pretty_print=True)
def testCompare(self, srefData, srunData, bVisual=False):
"""
Our comparison is very simple: same or different. N
We anyway return this in term of precision/recall
If we want to compute the error differently, we must define out own testInit testRecord, testReport
"""
dicTestByTask = dict()
# dicTestByTask['Names']= self.testFirstNameLastNameRecord(srefData, srunData,bVisual)
dicTestByTask['lastname'] = self.testRecordField(['lastname'], [None],
srefData, srunData,
bVisual)
dicTestByTask['firstname'] = self.testRecordField(['firstname'],
[None], srefData,
srunData, bVisual)
dicTestByTask['occupation'] = self.testRecordField(['occupation'],
[None], srefData,
srunData, bVisual)
dicTestByTask['location'] = self.testRecordField(['location'], [None],
srefData, srunData,
bVisual)
dicTestByTask['deathreason'] = self.testRecordField(['deathreason'],
[None], srefData,
srunData, bVisual)
dicTestByTask['names'] = self.testRecordField(
['firstname', 'lastname'], [None, None], srefData, srunData,
bVisual)
dicTestByTask['doktor'] = self.testRecordField(['doktor'],
['helfer_name'],
srefData, srunData,
bVisual)
# dicTestByTask['namedeathlocationoccupation']= self.testRecordField(['firstname','lastname','deathreason','location','occupation'],[None,None,None,None,None],srefData, srunData,bVisual)
dicTestByTask['situation'] = self.testRecordField(['situation'],
['family'], srefData,
srunData, bVisual)
# dicTestByTask['Year']= self.testYear(srefData, srunData,bVisual)
return dicTestByTask
def testRecordHtml(self, filename, data, nOk, nErr, nMiss):
if nOk == None:
assert nErr == None and nMiss == None, "INTERNAL ERROR"
#we are reporting on multiple tasks!!
lltisRefsRunbErrbMiss = data #this is a list of (taskName, nOk, nErr, nMiss, ltisRefsRunbErrbMiss)
else:
lltisRefsRunbErrbMiss = [(None, nOk, nErr, nMiss, data)]
#let's produce an HTML report!!
sCollecDir = os.path.dirname(self.testDirXML)
sCollec = os.path.basename(sCollecDir)
sFile = os.path.basename(self.getRefFileName(filename))[:-4]
sViewBaseUrl = "http://" #+ sHttpHost
fHtml = open(self.getHtmRunFileName(filename), "w", encoding='utf-8')
sCss = """
<style type="text/css">
.OK {
color: green;
}
.Error {
color: red;
}
.Error\+Miss {
color: darkred;
}
.Miss {
color: orange;
}
</style>
"""
sRpt = self.makeHTMLReportHeader(sViewBaseUrl, "dla_pdf", sCss,
sCollec + " - " + sFile,
sCollec + " - " + sFile)
fHtml.write(sRpt)
#sRpt += " Doc Prec. Recall F1\t nOk\t nErr\t nMiss\tFilename\n"
for taskName, nOk, nErr, nMiss, ltisRefsRunbErrbMiss in lltisRefsRunbErrbMiss:
if taskName == None: taskName = ""
sRpt = """
<hr/>
<h2>%s</h2>
<table>
<tr align="left">
<th></th>
<th>Page</th>
<th>Reference</th>
<th>Run</th>
<th></th>
</tr>
""" % taskName
fHtml.write(sRpt)
ipnum_prev = -1
key_prev = -1
for (key, ipnum, sRef, sRun, bErr, bMiss) in ltisRefsRunbErrbMiss:
if bErr and bMiss:
sRptType = "Error+Miss"
else:
if bErr:
sRptType = "Error"
elif bMiss:
sRptType = "Miss"
else:
sRptType = "OK"
sPfFile = sCollec + "/" + sFile + "/" + "pf%06d" % ipnum
srefenc = " ".join(x for x in sRef)
srun = " ".join(x for x in sRun)
# if ipnum > ipnum_prev: #a new page
if key != key_prev:
fHtml.write('<tr ><td>%s (%s)</td></tr>' % (key, ipnum))
fHtml.write(
'<tr class="%s"><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n'
% (
sRptType,
sRptType,
"" #ipnum
,
srefenc #sRef
,
srun #sRun
# , " - ".join(lsViews)
))
else: #some more results for the same pafe
fHtml.write(
'<tr class="%s"><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n'
% (sRptType, sRptType, "", srefenc, srun, ""))
ipnum_prev = ipnum
key_prev = key
fHtml.write('</table>')
fHtml.write('<p/>')
fHtml.write(
self.genHtmlTableReport(sCollec, None,
[(filename, nOk, nErr, nMiss, None)]))
fHtml.write('<hr>')
fHtml.close()
return
class columnDetection(Component.Component):
"""
build table column
"""
usage = ""
version = "v.01"
description = "description: column Detection"
#--- INIT -------------------------------------------------------------------------------------------------------------
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "columnDetection", self.usage,
self.version, self.description)
self.colname = None
self.docid = None
self.do2DS = False
self.THHighSupport = 0.75
# for --test
self.bCreateRef = False
self.evalData = None
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
# if dParams.has_key("coldir"):
# self.colname = dParams["coldir"]
if "docid" in dParams:
self.docid = dParams["docid"]
if "dsconv" in dParams:
self.do2DS = dParams["dsconv"]
self.bMining = dParams["mining"]
if "createref" in dParams:
self.bCreateRef = dParams["createref"]
def createTable(self, page):
"""
BB of all elements?
todo: Ignore O!
"""
x1, y1, x2, y2 = self.getBounbingBox(page)
if x1 is None: return None
myTable = XMLDSTABLEClass()
myTable.setX(x1)
myTable.setY(y1)
myTable.setWidth(x2 - x1)
myTable.setHeight(y2 - y1)
page.addObject(myTable)
return myTable
def processPage(self, page, emptyTable):
from util.XYcut import mergeSegments
### skrinking to be done: use center ?
# lCuts, _, _ = mergeSegments([(x.getX(),x.getX2(),x) for x in page.getAllNamedObjects(XMLDSTEXTClass)],0)
lCuts, _, _ = mergeSegments(
[(x.getX(), x.getX() + 0.25 * x.getWidth(), x)
for x in page.getAllNamedObjects(XMLDSTEXTClass)], 0)
# lCuts, _, _ = mergeSegments([(x.getX()+0.5*x.getWidth()-0.25*x.getWidth(),x.getX()+0.5*x.getWidth()+0.25*x.getWidth(),x) for x in page.getAllNamedObjects(XMLDSTEXTClass)],0)
for i, (x, _, cut) in enumerate(lCuts):
ll = list(cut)
ll.sort(key=lambda x: x.getY())
#add column
myCol = XMLDSTABLECOLUMNClass(i)
myCol.setPage(page)
myCol.setParent(emptyTable)
emptyTable.addObject(myCol)
myCol.setX(x)
myCol.setY(emptyTable.getY())
myCol.setHeight(emptyTable.getHeight())
if i + 1 < len(lCuts):
myCol.setWidth(lCuts[i + 1][0] - x)
else: # use table
myCol.setWidth(emptyTable.getX2() - x)
emptyTable.addColumn(myCol)
if not self.bMining:
myCol.tagMe(ds_xml.sCOL)
def getBounbingBox(self, page):
lElts = page.getAllNamedObjects(XMLDSTEXTClass)
if lElts == []: return None, None, None, None
lX1, lX2, lY1, lY2 = zip(*[(x.getX(), x.getX2(), x.getY(), x.getY2())
for x in lElts])
return min(lX1), min(lY1), max(lX2), max(lY2)
def findColumnsInDoc(self, lPages):
"""
find columns for each table in ODoc
"""
for page in lPages:
traceln("page: %d" % page.getNumber())
lch, lcv = self.mergeHorVerClusters(page)
# table = self.createTable(page)
# if table is not None:
# # table.tagMe(ds_xml.sTABLE)
# self.processPage(page,table)
def createContourFromListOfElements(self, lElts):
"""
create a polyline from a list of elements
input : list of elements
output: Polygon object
"""
from shapely.geometry import Polygon as pp
from shapely.ops import cascaded_union
lP = []
for elt in lElts:
sPoints = elt.getAttribute('points')
if sPoints is None:
lP.append(
pp([(elt.getX(), elt.getY()), (elt.getX(), elt.getY2()),
(elt.getX2(), elt.getY2()),
(elt.getX2(), elt.getY())]))
else:
lP.append(
pp([(float(x), float(y))
for x, y in zip(*[iter(sPoints.split(','))] * 2)]))
try:
ss = cascaded_union(lP)
except ValueError:
print(lElts, lP)
return None
return ss #list(ss.convex_hull.exterior.coords)
def mergeHorVerClusters(self, page):
"""
build Horizontal and vertical clusters
"""
from util import TwoDNeighbourhood as TwoDRel
lTexts = page.getAllNamedObjects(XMLDSTEXTClass)
for e in lTexts:
e.lright = []
e.lleft = []
e.ltop = []
e.lbottom = []
lVEdge = TwoDRel.findVerticalNeighborEdges(lTexts)
for a, b in lVEdge:
a.lbottom.append(b)
b.ltop.append(a)
for elt in lTexts:
# dirty!
elt.setHeight(max(5, elt.getHeight() - 3))
elt.setWidth(max(5, elt.getWidth() - 3))
TwoDRel.rotateMinus90degOLD(elt)
lHEdge = TwoDRel.findVerticalNeighborEdges(lTexts)
for elt in lTexts:
# elt.tagMe()
TwoDRel.rotatePlus90degOLD(elt)
# return
for a, b in lHEdge:
a.lright.append(b)
b.lleft.append(a)
# ss
for elt in lTexts:
elt.lleft.sort(key=lambda x: x.getX(), reverse=True)
# elt.lright.sort(key = lambda x:x.getX())
if len(elt.lright) > 1:
elt.lright = []
elt.lright.sort(key=lambda x: elt.signedRatioOverlapY(x),
reverse=True)
# print (elt, elt.getY(), elt.lright)
elt.ltop.sort(key=lambda x: x.getY())
if len(elt.lbottom) > 1:
elt.lbottom = []
elt.lbottom.sort(key=lambda x: elt.signedRatioOverlapX(x),
reverse=True)
lHClusters = []
# Horizontal
lTexts.sort(key=lambda x: x.getX())
lcovered = []
for text in lTexts:
if text not in lcovered:
# print ('START :', text, text.getContent())
lcovered.append(text)
lcurRow = [text]
curText = text
while curText is not None:
try:
nextT = curText.lright[0]
# print ('\t',[(x,curText.signedRatioOverlapY(x)) for x in curText.lright])
if nextT not in lcovered:
lcurRow.append(nextT)
lcovered.append(nextT)
curText = nextT
except IndexError:
curText = None
# lHClusters.append(lcurRow)
# print ("FINAL", list(map(lambda x:(x,x.getContent()),lcurRow)) )
if len(lcurRow) > 0:
# # create a contour for visualization
# # order by col: get top and bottom polylines for them
contour = self.createContourFromListOfElements(lcurRow)
lHClusters.append((lcurRow, contour))
# Vertical
lVClusters = []
lTexts.sort(key=lambda x: x.getY())
lcovered = []
for text in lTexts:
if text not in lcovered:
# print ('START :', text, text.getContent())
lcovered.append(text)
lcurCol = [text]
curText = text
while curText is not None:
try:
nextT = curText.lbottom[0]
# print ('\t',[(x,curText.signedRatioOverlapY(x)) for x in curText.lright])
if nextT not in lcovered and len(nextT.lbottom) == 1:
lcurCol.append(nextT)
lcovered.append(nextT)
curText = nextT
except IndexError:
curText = None
# # print ("FINAL", list(map(lambda x:(x,x.getContent()),lcurCol)) )
if len(lcurCol) > 0:
contour = self.createContourFromListOfElements(lcurCol)
lVClusters.append((lcurCol, contour))
if contour:
# print (contour.bounds)
r = XMLDSObjectClass()
r.setName('cc')
r.setParent(page)
# r.addAttribute('points',spoints)
x1, y1, x2, y2 = contour.bounds
r.setXYHW(x1, y1, y2 - y1, x2 - x1)
page.addObject(r)
# r.tagMe('BLOCK')
print(page.getAllNamedObjects('cc'))
return lHClusters, lVClusters
def documentMining(self, lPages):
"""
need to clean up REGION nodes
"""
seqMiner = tableColumnMiner()
seqMiner.columnMining(lPages, self.THHighSupport, sTag=ds_xml.sCOL)
def checkInputFormat(self, lPages):
"""
delete regions : copy regions elements at page object
unlink subnodes
"""
for page in lPages:
lRegions = page.getAllNamedObjects("REGION")
lElts = []
[lElts.extend(x.getObjects()) for x in lRegions]
[page.addObject(x, bDom=True) for x in lElts]
[page.removeObject(x, bDom=True) for x in lRegions]
def run(self, doc):
"""
load dom and find rows
"""
# conver to DS if needed
if self.bCreateRef:
if self.do2DS:
dsconv = primaAnalysis()
doc = dsconv.convert2DS(doc, self.docid)
refdoc = self.createRef(doc)
return refdoc
# single ref per page
refdoc = self.createRefPerPage(doc)
return None
if self.do2DS:
dsconv = primaAnalysis()
self.doc = dsconv.convert2DS(doc, self.docid)
else:
self.doc = doc
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
self.checkInputFormat(self.lPages)
self.findColumnsInDoc(self.lPages)
if self.bMining:
self.documentMining(self.lPages)
if self.bCreateRef:
refdoc = self.createRef(self.doc)
return refdoc
# if self.do2DS:
# # bakc to PageXml
# conv= DS2PageXMLConvertor()
# lPageXDoc = conv.run(self.doc)
# conv.storeMultiPageXml(lPageXDoc,self.getOutputFileName())
# print self.getOutputFileName()
# return None
return self.doc
################ TEST ##################
def testRun(self, filename, outFile=None):
"""
evaluate using ABP new table dataset with tablecell
"""
self.evalData = None
doc = self.loadDom(filename)
doc = self.run(doc)
self.evalData = self.createRef(doc)
if outFile: self.writeDom(doc)
# return self.evalData.serialize('utf-8',1)
return etree.tostring(self.evalData,
encoding='unicode',
pretty_print=True)
def overlapX(self, zone):
[a1, a2] = self.getX(), self.getX() + self.getWidth()
[b1, b2] = zone.getX(), zone.getX() + zone.getWidth()
return min(a2, b2) >= max(a1, b1)
def overlapY(self, zone):
[a1, a2] = self.getY(), self.getY() + self.getHeight()
[b1, b2] = zone.getY(), zone.getY() + zone.getHeight()
return min(a2, b2) >= max(a1, b1)
def signedRatioOverlap(self, z1, z2):
"""
overlap self and zone
return surface of self in zone
"""
[x1, y1, h1, w1] = z1.getX(), z1.getY(), z1.getHeight(), z1.getWidth()
[x2, y2, h2, w2] = z2.getX(), z2.getY(), z2.getHeight(), z2.getWidth()
fOverlap = 0.0
if self.overlapX(z2) and self.overlapY(z2):
[x11, y11, x12, y12] = [x1, y1, x1 + w1, y1 + h1]
[x21, y21, x22, y22] = [x2, y2, x2 + w2, y2 + h2]
s1 = w1 * h1
# possible ?
if s1 == 0: s1 = 1.0
#intersection
nx1 = max(x11, x21)
nx2 = min(x12, x22)
ny1 = max(y11, y21)
ny2 = min(y12, y22)
h = abs(nx2 - nx1)
w = abs(ny2 - ny1)
inter = h * w
if inter > 0:
fOverlap = inter / s1
else:
# if overX and Y this is not possible !
fOverlap = 0.0
return fOverlap
def findSignificantOverlap(self, TOverlap, ref, run):
"""
return
"""
pref, rowref = ref
prun, rowrun = run
if pref != prun: return False
return rowref.ratioOverlap(rowrun) >= TOverlap
def testCPOUM(self, TOverlap, srefData, srunData, bVisual=False):
"""
TOverlap: Threshols used for comparing two surfaces
Correct Detections:
under and over segmentation?
"""
cntOk = cntErr = cntMissed = 0
RefData = etree.XML(srefData.strip("\n").encode('utf-8'))
RunData = etree.XML(srunData.strip("\n").encode('utf-8'))
# try:
# RunData = libxml2.parseMemory(srunData.strip("\n"), len(srunData.strip("\n")))
# except:
# RunData = None
# return (cntOk, cntErr, cntMissed)
lRun = []
if RunData:
lpages = RunData.xpath('//%s' % ('PAGE'))
for page in lpages:
pnum = page.get('number')
#record level!
lRows = page.xpath(".//%s" % ("ROW"))
lORows = map(lambda x: XMLDSTABLECOLUMNClass(0, x), lRows)
for row in lORows:
row.fromDom(row._domNode)
row.setIndex(row.getAttribute('id'))
lRun.append((pnum, row))
print(lRun)
lRef = []
lPages = RefData.xpath('//%s' % ('PAGE'))
for page in lPages:
pnum = page.get('number')
lRows = page.xpath(".//%s" % ("ROW"))
lORows = map(lambda x: XMLDSTABLECOLUMNClass(0, x), lRows)
for row in lORows:
row.fromDom(row._domNode)
row.setIndex(row.getAttribute('id'))
lRef.append((pnum, row))
refLen = len(lRef)
# bVisual = True
ltisRefsRunbErrbMiss = list()
lRefCovered = []
for i in range(0, len(lRun)):
iRef = 0
bFound = False
bErr, bMiss = False, False
runElt = lRun[i]
# print '\t\t===',runElt
while not bFound and iRef <= refLen - 1:
curRef = lRef[iRef]
if runElt and curRef not in lRefCovered and self.findSignificantOverlap(
TOverlap, runElt, curRef):
bFound = True
lRefCovered.append(curRef)
iRef += 1
if bFound:
if bVisual: print("FOUND:", runElt, ' -- ', lRefCovered[-1])
cntOk += 1
else:
curRef = ''
cntErr += 1
bErr = True
if bVisual: print("ERROR:", runElt)
if bFound or bErr:
ltisRefsRunbErrbMiss.append(
(int(runElt[0]), curRef, runElt, bErr, bMiss))
for i, curRef in enumerate(lRef):
if curRef not in lRefCovered:
if bVisual: print("MISSED:", curRef)
ltisRefsRunbErrbMiss.append(
(int(curRef[0]), curRef, '', False, True))
cntMissed += 1
ltisRefsRunbErrbMiss.sort(key=lambda xyztu: xyztu[0])
# print cntOk, cntErr, cntMissed,ltisRefsRunbErrbMiss
return (cntOk, cntErr, cntMissed, ltisRefsRunbErrbMiss)
def testCompare(self, srefData, srunData, bVisual=False):
"""
as in Shahad et al, DAS 2010
Correct Detections
Partial Detections
Over-Segmented
Under-Segmented
Missed
False Positive
"""
dicTestByTask = dict()
dicTestByTask['T50'] = self.testCPOUM(0.50, srefData, srunData,
bVisual)
# dicTestByTask['T75']= self.testCPOUM(0.750,srefData,srunData,bVisual)
# dicTestByTask['T100']= self.testCPOUM(0.50,srefData,srunData,bVisual)
# dicTestByTask['FirstName']= self.testFirstNameRecord(srefData, srunData,bVisual)
# dicTestByTask['Year']= self.testYear(srefData, srunData,bVisual)
return dicTestByTask
def createColumnsWithCuts(self, lXCuts, table, tableNode, bTagDoc=False):
"""
create column dom node
"""
prevCut = None
lXCuts.sort()
for index, cut in enumerate(lXCuts):
# first correspond to the table: no rpw
if prevCut is not None:
colNode = etree.Element("COL")
tableNode.append(colNode)
colNode.set('x', str(prevCut))
colNode.set('width', "{:.2f}".format(cut - prevCut))
colNode.set('y', str(table.getY()))
colNode.set('height', str(table.getHeight()))
colNode.set('id', str(index - 1))
prevCut = cut
#last
cut = table.getX2()
colNode = etree.Element("COL")
tableNode.append(colNode)
colNode.set('x', "{:.2f}".format(prevCut))
colNode.set('width', "{:.2f}".format(cut - prevCut))
colNode.set('y', str(table.getY()))
colNode.set('height', str(table.getHeight()))
colNode.set('id', str(index))
def createRef(self, doc):
"""
create a ref file from the xml one
"""
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(doc,
listPages=range(self.firstPage,
self.lastPage + 1))
root = etree.Element("DOCUMENT")
refdoc = etree.ElementTree(root)
for page in self.ODoc.getPages():
#imageFilename="..\col\30275\S_Freyung_021_0001.jpg" width="977.52" height="780.0">
pageNode = etree.Element('PAGE')
pageNode.set("number", page.getAttribute('number'))
pageNode.set("pagekey",
os.path.basename(page.getAttribute('imageFilename')))
pageNode.set("width", page.getAttribute('width'))
pageNode.set("height", page.getAttribute('height'))
root.append(pageNode)
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
dCol = {}
tableNode = etree.Element('TABLE')
tableNode.set("x", table.getAttribute('x'))
tableNode.set("y", table.getAttribute('y'))
tableNode.set("width", table.getAttribute('width'))
tableNode.set("height", table.getAttribute('height'))
pageNode.append(tableNode)
for cell in table.getAllNamedObjects(XMLDSTABLECELLClass):
try:
dCol[int(cell.getAttribute("col"))].append(cell)
except KeyError:
dCol[int(cell.getAttribute("col"))] = [cell]
lXcuts = []
for colid in sorted(dCol.keys()):
lXcuts.append(
min(list(map(lambda x: x.getX(), dCol[colid]))))
self.createColumnsWithCuts(lXcuts, table, tableNode)
return refdoc
class tableRowMiner(Component.Component):
"""
tableRowMiner class: a component to mine table to find out horizontal cuts (hence rows)
"""
#DEFINE the version, usage and description of this particular component
usage = ""
version = "v.01"
description = "description: table row miner "
#--- INIT -------------------------------------------------------------------------------------------------------------
def __init__(self):
"""
Always call first the Component constructor.
"""
Component.Component.__init__(self, "tableRowMiner", self.usage,
self.version, self.description)
# TH for comparing numerical features
## need to be fucntion of leading: when leading small THNUMERICAL small, lineHeighr reduced a well?
self.THNUMERICAL = 25
# use for evaluation
self.THCOMP = 10
self.evalData = None
self.THHighSupport = 0.33
self.bManual = False
def setParams(self, dParams):
"""
Always call first the Component setParams
Here, we set our internal attribute according to a possibly specified value (otherwise it stays at its default value)
"""
Component.Component.setParams(self, dParams)
if "pattern" in dParams:
self.manualPattern = eval(dParams["pattern"])
self.bManual = True
if "thhighsupport" in dParams:
self.THHighSupport = dParams["thhighsupport"] * 0.01
def testHighSupport(self, sequences, th):
"""
compute unigram support
"""
# from mssp
from collections import Counter
import itertools
sequence_count = len(sequences)
flattened_sequences = [
list(set(itertools.chain(*sequence))) for sequence in sequences
]
support_counts = dict(
Counter(item for flattened_sequence in flattened_sequences
for item in flattened_sequence))
actual_supports = {
item: support_counts.get(item) / float(sequence_count)
for item in support_counts.keys()
}
# lOneSupport= [k for k,v in actual_supports.iteritems() if v >= 0.5 ]
lOneSupport = [k for k, v in actual_supports.items() if v >= th]
# print(actual_supports.items() )
# print (th,lOneSupport)
return lOneSupport
def createFeatureFromValue(self, elt, value, name):
feature = featureObject()
feature.setName(name)
feature.setTH(self.THNUMERICAL)
feature.addNode(elt)
feature.setObjectName(elt)
feature.setValue(float(value))
feature.setType(featureObject.NUMERICAL)
return feature
def columnMining(self, table, thnum, th, predefinedCuts=[]):
"""
for a table: take itemset=colmun, item=cell(Y) + separator
- test: is a same rowgrid for all pages: row of fixed positions, size
separator
PREPRO: get the Y of overlaping separators for each col
"""
if thnum != None:
self.THNUMERICAL = thnum
lElts = table.getColumns() #getAllNamedObjects(XMLDSTABLECOLUMNClass)
for elt in lElts:
# how to add separator?
# for c in elt.getCells():c.setHeight()
elt.resetFeatures()
elt.setFeatureFunction(elt.getSetOfListedAttributes,
self.THNUMERICAL,
lFeatureList=['y'],
myLevel=XMLDSTABLECELLClass)
## add predefinedCuts here
elt.computeSetofFeatures()
for prevFea in predefinedCuts:
f = self.createFeatureFromValue(elt, round(prevFea), 'y')
elt.addFeature(f)
seqGen = sequenceMiner()
seqGen.bDebug = False
seqGen.setMaxSequenceLength(1)
# seqGen.setSDC(0.7) # related to noise level AND STRUCTURES (if many columns)
# _ = seqGen.featureGeneration(lElts,2) # more at token level, but at text level: freq=2
seqGen.setObjectLevel(XMLDSTABLECELLClass)
for elt in lElts:
elt.lFeatureForParsing = elt.getSetofFeatures()
elt.lFeatureForParsing.sort(key=lambda x: x.getValue())
# print( elt, elt.lFeatureForParsing)
#
lSortedFeatures = seqGen.featureGeneration(lElts, 2)
# for f in lSortedFeatures:
# print ("%s\s%s"%(f, f.getNodes()))
lmaxSequence = seqGen.generateItemsets(lElts)
# for elt in lElts:
# print elt, elt.getCanonicalFeatures()
lSeq, _ = seqGen.generateMSPSData(lmaxSequence,
lSortedFeatures,
mis=0.5)
lOneSupport = self.testHighSupport(lSeq, th)
lOneSupport.sort(key=lambda x: x.getValue())
return lOneSupport
# lTerminalTemplates=[]
# lCurList=lElts[:]
# lCurList,lTerminalTemplates = self.mineLineFeature(seqGen,lCurList,lTerminalTemplates)
# print lTerminalTemplates
# return
def mainMining(self, lPages):
"""
mine with incremental length
"""
import util.TwoDNeighbourhood as TwoDRel
lLElts = [[] for i in range(0, len(lPages))]
for i, page in enumerate(lPages):
lElts = page.getAllNamedObjects(
XMLDSTABLECELLClass
) #+page.getAllNamedObjects(XMLDSGRAPHLINEClass)
for e in lElts:
e.lnext = []
## filter elements!!!
lElts = filter(lambda x: min(x.getHeight(), x.getWidth()) > 10,
lElts)
lElts = filter(lambda x: x.getHeight() > 10, lElts)
lElts.sort(key=lambda x: x.getY())
lLElts[i] = lElts
for elt in lElts:
TwoDRel.rotateMinus90deg(
elt
) #rotate by 90 degrees and look for vertical neighbors :-)
lHEdge = TwoDRel.findVerticalNeighborEdges(lElts)
for elt in lElts:
TwoDRel.rotatePlus90deg(elt)
# lVEdge = TwoDRel.findVerticalNeighborEdges(lElts)
for a, b in lHEdge:
a.lnext.append(b)
for i, page, in enumerate(lPages):
lElts = lLElts[i]
for elt in lElts:
elt.setFeatureFunction(elt.getSetOfListedAttributes,
self.THNUMERICAL,
lFeatureList=['y'],
myLevel=XMLDSTABLECELLClass)
elt.computeSetofFeatures()
# print elt.getSetofFeatures()
seqGen = sequenceMiner()
seqGen.setMaxSequenceLength(1)
seqGen.setSDC(
0.7
) # related to noise level AND STRUCTURES (if many columns)
_ = seqGen.featureGeneration(
lElts, 2) # more at token level, but at text level: freq=2
seqGen.setObjectLevel(XMLDSTABLECELLClass)
# lKleendPlus = self.getKleenePlusFeatures(lElts)
# print lKleendPlus
for elt in lElts:
elt.lFeatureForParsing = elt.getSetofFeatures()
# print elt, elt.lFeatureForParsing
#
lTerminalTemplates = []
lCurList = lElts[:]
lCurList, lTerminalTemplates = self.mineLineFeature(
seqGen, lCurList, lTerminalTemplates)
# print lTerminalTemplates
for mytemplate in lTerminalTemplates:
page.addVerticalTemplate(mytemplate)
page.addVSeparator(mytemplate, mytemplate.getPattern())
del seqGen
# self.tagAsRegion(lPages)
def mineLineFeature(self, seqGen, lCurList, lTerminalTemplates):
"""
get a set of lines and mine them
"""
seqGen.setMinSequenceLength(1)
seqGen.setMaxSequenceLength(1)
print('***' * 20)
seqGen.bDebug = False
for elt in lCurList:
if elt.getSetofFeatures() is None:
elt.resetFeatures()
elt.setFeatureFunction(elt.getSetOfListedAttributes,
self.THNUMERICAL,
lFeatureList=['virtual'],
myLevel=XMLDSTABLECELLClass)
elt.computeSetofFeatures()
elt.lFeatureForParsing = elt.getSetofFeatures()
else:
elt.setSequenceOfFeatures(elt.lFeatureForParsing)
# print elt, elt.getSetofFeatures()
lSortedFeatures = seqGen.featureGeneration(lCurList, 2)
for cf in lSortedFeatures:
cf.setWeight(len((cf.getNodes())))
lmaxSequence = seqGen.generateItemsets(lCurList)
seqGen.bDebug = False
lSeq, _ = seqGen.generateMSPSData(lmaxSequence,
lSortedFeatures + lTerminalTemplates,
mis=0.01)
lPatterns = seqGen.miningSequencePrefixScan(lSeq)
# lPatterns = seqGen.beginMiningSequences(lSeq,lSortedFeatures,lMIS)
if lPatterns is None:
return lCurList, lTerminalTemplates
lPatterns.sort(key=lambda xy: xy[0], reverse=True)
print("List of patterns and their support:")
for p, support in lPatterns:
if support >= 1:
print(p, support)
seqGen.THRULES = 0.95
lSeqRules = seqGen.generateSequentialRules(lPatterns)
" here store features which are redundant and consider only the core feature"
dTemplatesCnd = self.analyzeListOfPatterns(lPatterns, {})
lFullTemplates, lTerminalTemplates, tranprob = seqGen.testTreeKleeneageTemplates(
dTemplatesCnd, lCurList, iterMax=40)
return lCurList, lTerminalTemplates
def selectFinalTemplates(self, lTemplates, transProb, lElts):
"""
apply viterbi to select best sequence of templates
"""
import spm.viterbi as viterbi
if lTemplates == []:
return None
def buildObs(lTemplates, lElts):
"""
build observation prob
"""
N = len(lTemplates) + 1
obs = np.zeros((N, len(lElts)), dtype=np.float16) + 10e-3
for i, temp in enumerate(lTemplates):
for j, elt in enumerate(lElts):
# how to dela with virtual nodes
try:
_, _, score = temp.registration(elt)
except:
score = 1
if score == -1:
score = 0.0
obs[i, j] = score
if np.isinf(obs[i, j]):
obs[i, j] = 64000
if np.isnan(obs[i, j]):
obs[i, j] = 0.0
# print i,j,elt,elt.lX, temp,score
#add no-template:-1
return obs / np.amax(obs)
N = len(lTemplates) + 1
initialProb = np.ones(N)
initialProb = np.reshape(initialProb, (N, 1))
obs = buildObs(lTemplates, lElts)
np.set_printoptions(precision=3, linewidth=1000)
# print "transProb"
# print transProb
# print
# print obs
d = viterbi.Decoder(initialProb, transProb, obs)
states, score = d.Decode(np.arange(len(lElts)))
# add empty template (last one in state)
lTemplates.append(None)
print(states, score)
#assign to each elt the template assigned by viterbi
for i, elt, in enumerate(lElts):
# try: print elt,elt.lX, lTemplates[states[i]]
# except: print elt, elt.lX, 'no template'
mytemplate = lTemplates[states[i]]
elt.resetTemplate()
if mytemplate is not None:
elt.addTemplate(mytemplate)
try:
registeredPoints, lMissing, score = mytemplate.registration(
elt)
except:
registeredPoints = None
if registeredPoints:
# print registeredPoints, lMissing , score
if lMissing != []:
registeredPoints.extend(zip(lMissing, lMissing))
registeredPoints.sort(key=lambda xy: xy[0].getValue())
lcuts = map(lambda refcut: refcut[1], registeredPoints)
## store features for the final parsing!!!
# print elt, lcuts
# elt.addVSeparator(mytemplate,lcuts)
# return the new list with kleenePlus elts for next iteration
## reparse ?? YES using the featureSet given by viterbi -> create an objectClass per kleenePlus element: objects: sub tree
# print lTemplates[0]
# self.parseWithTemplate(lTemplates[0], lElts)
# elt = template
return score
def getKleenePlusFeatures(self, lElts):
"""
select KleenePlus elements based on .next (only possible for unigrams)
"""
dFreqFeatures = {}
dKleenePlusFeatures = {}
lKleenePlus = []
for elt in lElts:
for fea in elt.getSetofFeatures():
try:
dFreqFeatures[fea] += 1
except KeyError:
dFreqFeatures[fea] = 1
for nextE in elt.lnext:
if fea in nextE.getSetofFeatures():
try:
dKleenePlusFeatures[fea].append((elt, nextE))
except KeyError:
dKleenePlusFeatures[fea] = [(elt, nextE)]
for fea in dFreqFeatures:
try:
dKleenePlusFeatures[fea]
if len(dKleenePlusFeatures[fea]) >= 0.5 * dFreqFeatures[fea]:
lKleenePlus.append(fea)
except:
pass
return lKleenePlus
def computePatternScore(self, pattern):
"""
consider the frequency of the pattern and the weights of the features
"""
fScore = 0
#terminal
if not isinstance(pattern, list):
fScore += pattern.getCanonical().getWeight()
else:
for child in pattern:
fScore += self.computePatternScore(child)
# print 'score:',pattern ,fScore
return fScore
def analyzeListOfPatterns(
self,
lPatterns,
dCA,
):
"""
select patterns with no ancestor
other criteria ?
if many with similar frequency: sort using computePatternScore?
"""
# reorder lPatterns considering feature weights and # of elements (for equally frequent patterns)
# lPatterns.sort(key=lambda (x,y):self.computePatternScore(x),reverse=True)
# for x,y in lPatterns:
# print x,y,self.computePatternScore(x)
dTemplatesTypes = {}
for pattern, support in filter(lambda xy: xy[1] > 1, lPatterns):
try:
dCA[str(pattern)]
bSkip = True
except KeyError:
bSkip = False
# if step > 0 and len(pattern) == 1:
# bSkip=True
if not bSkip:
template = treeTemplateClass()
template.setPattern(pattern)
template.buildTreeFromPattern(pattern)
template.setType('lineTemplate')
try:
dTemplatesTypes[template.__class__.__name__].append(
(pattern, support, template))
except KeyError:
dTemplatesTypes[template.__class__.__name__] = [
(pattern, support, template)
]
return dTemplatesTypes
def processWithTemplate(self, lPattern, lPages):
"""
process sequence of pqges with given pattern
create table
"""
lfPattern = []
for itemset in lPattern:
fItemset = []
for item in itemset:
f = featureObject()
f.setName("x")
f.setType(featureObject.NUMERICAL)
f.setValue(item)
f.setTH(self.THNUMERICAL)
fItemset.append(f)
lfPattern.append(fItemset)
pattern = lfPattern
print(pattern)
### in prodf: mytemplate given by page.getVerticalTemplates()
mytemplate = treeTemplateClass()
mytemplate.setPattern(pattern[0])
# registration provides best matching
## from registration matched: select the final cuts
for i, p in enumerate(lPages):
p.lFeatureForParsing = p.lf_XCut
sys.stdout.flush()
registeredPoints1, lMissing1, score1 = mytemplate.registration(p)
if score1 >= 0:
lfinalCuts = map(
lambda xy: xy[1],
filter(lambda xy: xy[0] != 'EMPTY', registeredPoints1))
print(p, 'final1:', lfinalCuts, lMissing1)
p.addVerticalTemplate(mytemplate)
p.addVSeparator(mytemplate, lfinalCuts)
else:
print('NO REGISTRATION')
self.tagAsRegion(lPages)
return 1
def tagAsRegion(self, lPages):
"""
create regions
"""
for page in lPages:
if page.getNode():
# if several template ???
for template in page.getVerticalTemplates():
page.getdVSeparator(template).sort(
key=lambda x: x.getValue())
print(page.getdVSeparator(template))
page.getNode().setProp('template',
str(page.getdVSeparator(template)))
XMinus = 1
prevcut = 10
# print page, page.getdVSeparator(template)
for cut in page.getdVSeparator(template):
cellNode = etree.Element('REGION')
cellNode.set("y", str(prevcut))
## it is better to avoid
YMinus = 10
cellNode.set("x", str(XMinus))
cellNode.set("width",
str(page.getWidth() - 2 * YMinus))
cellNode.set("height", str(cut.getValue() - prevcut))
page.getNode().addChild(cellNode)
prevcut = cut.getValue()
def generateTestOutput(self, lPages):
"""
create a run XML file
"""
root = etree.Element('DOCUMENT')
self.evalData = etree.ElementTree(root)
for page in lPages:
domp = etree.Element('PAGE')
domp.set('number', page.getAttribute('number'))
root.addChild(domp)
for sep in page.lVSeparator:
print(page.lVSeparator)
domsep = etree.Element('SeparatorRegion')
domp.append(domsep)
# domsep.setProp('x', str(sep[0].getValue()))
domsep.set('x', str(sep[0]))
#--- RUN ---------------------------------------------------------------------------------------------------------------
def run(self, doc):
"""
take a set of line in a page and mine it
"""
self.doc = doc
# use the lite version
self.ODoc = XMLDSDocument()
self.ODoc.loadFromDom(self.doc,
listPages=range(self.firstPage,
self.lastPage + 1))
self.lPages = self.ODoc.getPages()
if self.bManual:
self.processWithTemplate(self.manualPattern, self.lPages)
else:
# self.mainMining(self.lPages)
for page in self.lPages:
print("page")
lTables = page.getAllNamedObjects(XMLDSTABLEClass)
for table in lTables:
self.columnMining(table)
self.addTagProcessToMetadata(self.doc)
return self.doc