def storeMultiPageXml(self,lListDocs,outputFileName=None): """ write a multipagePageXml file """ from xml_formats.PageXml import MultiPageXml mp = MultiPageXml() newDoc = mp.makeMultiPageXmlMemory(list(map(lambda xy:xy[0],lListDocs))) ## print(outputFileName) if outputFileName is None: outputFileName = os.path.dirname(self.inputFileName) + os.sep + ".."+os.sep +"col" + os.sep + os.path.basename(self.inputFileName)[:-7] + "_du.mpxml" # print(outputFileName) res= newDoc.write(outputFileName, encoding="UTF-8",pretty_print=True,xml_declaration=True)
def test_DS2PageXmlConversion(): filename = os.path.join(sTESTS_DIR, 'testDS2PageXml/RRB_MM_01_033_Jahr_1810.ds.xml') conv= DS2PageXMLConvertor() conv.inputFileName = filename doc = conv.loadDom(filename) lPageXmlDocs = conv.run(doc) mp = MultiPageXml() # newDoc = mp.makeMultiPageXmlMemory(map(lambda (x,y):x,lPageXmlDocs)) newDoc = mp.makeMultiPageXmlMemory([x for x,_y in lPageXmlDocs]) newDoc.write(os.path.join(sTESTS_DIR, "testDS2PageXml/RRB_MM_01_033_Jahr_1810.mpxml"), xml_declaration=True, encoding="UTF-8", pretty_print=True)
def processDocument(self, coldir, colid, docid, dom=None): """ process a single document 1 python ../../src/xml_formats/PageXml.py trnskrbs_5400/col/17442 --ext=pxml 2 python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form 3 python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400 4 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442 5 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml 6 python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442 7 python ../../../TranskribusPyClient/src/TranskribusCommands/do_htrRnn.py <model-name> <dictionary-name> 5400 17442 wait 8 python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force #covnert to ds 9 python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442 10 python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml --doie --usetemplate """ #create Transkribus client self.myTrKCient = TranskribusClient(sServerUrl=self.server, proxies={}, loggingLevel=logging.WARN) #login _ = self.login(self.myTrKCient, trace=trace, traceln=traceln) # self.downloadCollection(colid,coldir,docid,bNoImg=False,bForce=True) ## load dom if dom is None: self.inputFileName = os.path.abspath( os.path.join(coldir, TableProcessing.sCOL, docid + TableProcessing.sMPXMLExtension)) mpxml_doc = self.loadDom() nbPages = MultiPageXml.getNBPages(mpxml_doc) else: # load provided mpxml mpxml_doc = dom nbPages = MultiPageXml.getNBPages(mpxml_doc) # ### table registration: need to compute/select??? the template # # perform LA separator, table registration, baseline with normalization # #python ../../src/tasks/performCVLLA.py --coldir=trnskrbs_5400/ --docid=17442 -i trnskrbs_5400/col/17442.mpxml --bl --regTL --form # tableregtool= LAProcessor() # # latool.setParams(dParams) # tableregtool.coldir = coldir # tableregtool.docid = docid # tableregtool.bTemplate, tableregtool.bSeparator , tableregtool.bBaseLine , tableregtool.bRegularTextLine = True,False,False,False # # creates xml and a new mpxml # mpxml_doc,nbPages = tableregtool.performLA(mpxml_doc) # # # self.upLoadDocument(colid, coldir,docid,sNote='NLE workflow;table reg done') lJobIDs = self.applyLA_URO(colid, docid, nbPages) return bWait = True assert lJobIDs != [] jobid = lJobIDs[-1] traceln("waiting for job %s" % jobid) while bWait: dInfo = self.myTrKCient.getJobStatus(jobid) bWait = dInfo['state'] not in ['FINISHED', 'FAILED'] ## coldir??? self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True) ##STOP HERE FOR DAS newx testset: return # tag text for BIES cell #python ../../src/tasks/DU_ABPTable_T.py modelMultiType tableRow2 --run=trnskrbs_5400 """ needed : doer = DU_ABPTable_TypedCRF(sModelName, sModelDir, """ doer = DU_ABPTable_TypedCRF(self.sRowModelName, self.sRowModelDir) doer.load() ## needed predict at file level, and do not store dom, but return it rowpath = os.path.join(coldir, "col") BIESFiles = doer.predict([rowpath], docid) BIESDom = self.loadDom(BIESFiles[0]) # res= BIESDom.saveFormatFileEnc('test.mpxml', "UTF-8",True) # MPXML2DS #python ../../src/xml_formats/Page2DS.py --pattern=trnskrbs_5400/col/17442_du.mpxml -o trnskrbs_5400/xml/17442.ds_xml --docid=17442 dsconv = primaAnalysis() DSBIESdoc = dsconv.convert2DS(BIESDom, self.docid) # create XMLDOC object self.ODoc = XMLDSDocument() self.ODoc.loadFromDom( DSBIESdoc) #,listPages = range(self.firstPage,self.lastPage+1)) # create row #python src/IE_test.py -i trnskrbs_5400/xml/17442.ds_xml -o trnskrbs_5400/out/17442.ds_xml rdc = RowDetection() rdc.findRowsInDoc(self.ODoc) #python ../../src/xml_formats/DS2PageXml.py -i trnskrbs_5400/out/17442.ds_xml --multi # DS2MPXML DS2MPXML = DS2PageXMLConvertor() lPageXml = DS2MPXML.run(self.ODoc.getDom()) if lPageXml != []: # if DS2MPXML.bMultiPages: newDoc = MultiPageXml.makeMultiPageXmlMemory( map(lambda xy: xy[0], lPageXml)) outputFileName = os.path.join( self.coldir, sCOL, self.docid + TableProcessing.sMPXMLExtension) newDoc.write(outputFileName, xml_declaration=True, encoding="UTF-8", pretty_print=True) # else: # DS2MPXML.storePageXmlSetofFiles(lPageXml) return #upload # python ../../../TranskribusPyClient/src/TranskribusCommands/TranskribusDU_transcriptUploader.py --nodu trnskrbs_5400 5400 17442 self.upLoadDocument(colid, coldir, docid, sNote='NLE workflow;table row done') ## apply HTR ## how to deal with specific dictionaries? ## here need to know the ontology and the template nbPages = 1 jobid = self.applyHTR(colid, docid, nbPages, self.sHTRmodel, self.sDictName) bWait = True traceln("waiting for job %s" % jobid) while bWait: dInfo = self.myTrKCient.getJobStatus(jobid) bWait = dInfo['state'] not in ['FINISHED', 'FAILED', 'CANCELED'] # download where??? # python ../../../TranskribusPyClient/src/TranskribusCommands/Transkribus_downloader.py 5400 --force # coldir is not right!! coldir must refer to the parent folder! self.downloadCollection(colid, coldir, docid, bNoImg=True, bForce=True)