) # ntA.setXpathExpr( (".//pc:TextLine | .//pc:TextRegion" #how to find the nodes # , "./pc:TextEquiv") #how to get their text # ) DU_GRAPH.addNodeType(nt) return DU_GRAPH if __name__ == "__main__": # import better_exceptions # better_exceptions.MAX_LENGTH = None # standard command line options for CRF- ECN- GAT-based methods usage, parser = DU_Task_Factory.getStandardOptionsParser(sys.argv[0]) traceln("VERSION: %s" % DU_Task_Factory.getVersion()) # --- #parse the command line (options, args) = parser.parse_args() try: sModelDir, sModelName = args except Exception as e: traceln("Specify a model folder and a model name!") DU_Task_Factory.exit(usage, 1, e) doer = DU_Task_Factory.getDoer(sModelDir, sModelName , options = options
def main(sys_argv_0, sLabelAttribute, cNodeType=My_ConjugateNodeType): def getConfiguredGraphClass(_doer): """ In this class method, we must return a configured graph class """ # each graph reflects 1 page if options.bSeparator: DU_GRAPH = ConjugateSegmenterGraph_MultiSinglePageXml_Separator else: DU_GRAPH = ConjugateSegmenterGraph_MultiSinglePageXml ntClass = cNodeType nt = ntClass(sLabelAttribute #some short prefix because labels below are prefixed with it , [] # in conjugate, we accept all labels, andNone becomes "none" , [] , False # unused , BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3)) #we reduce overlap in this way ) nt.setLabelAttribute(sLabelAttribute) nt.setXpathExpr( (".//pc:TextLine" #how to find the nodes #, "./pc:TextEquiv") #how to get their text , ".//pc:Unicode") #how to get their text ) DU_GRAPH.addNodeType(nt) return DU_GRAPH # standard command line options for CRF- ECN- GAT-based methods usage, parser = DU_Task_Factory.getStandardOptionsParser(sys_argv_0) parser.add_option("--separator", dest='bSeparator', action="store_true" , default=False, help="Use the graphical spearators, if any, as edge features.") parser.add_option("--text" , dest='bText' , action="store_true" , default=False, help="Use textual information if any, as node and edge features.") parser.add_option("--edge_vh", "--edge_hv" , dest='bShift' , action="store_true" , default=False, help="Shift edge feature by range depending on edge type.") parser.add_option("--jsonocr", dest='bJsonOcr', action="store_true" , help="I/O is in json") traceln("VERSION: %s" % DU_Task_Factory.getVersion()) # --- #parse the command line (options, args) = parser.parse_args() try: sModelDir, sModelName = args except Exception as e: traceln("Specify a model folder and a model name!") DU_Task_Factory.exit(usage, 1, e) if options.bText : traceln(" - using textual data, if any") if options.bSeparator: traceln(" - using graphical separators, if any") if options.bShift : traceln(" - shift edge features by edge type") if options.bText: if options.bSeparator: if options.bShift: cFeatureDefinition = Features_June19_Full_Separator_Shift else: cFeatureDefinition = Features_June19_Full_Separator else: if options.bShift: cFeatureDefinition = Features_June19_Full_Shift else: cFeatureDefinition = Features_June19_Full else: if options.bSeparator: if options.bShift: cFeatureDefinition = Features_June19_Simple_Separator_Shift else: cFeatureDefinition = Features_June19_Simple_Separator else: if options.bShift: cFeatureDefinition = Features_June19_Simple_Shift else: cFeatureDefinition = Features_June19_Simple # === SETTING the graph type (and its node type) a,d the feature extraction pipe doer = DU_Task_Factory.getDoer(sModelDir, sModelName , options = options , fun_getConfiguredGraphClass= getConfiguredGraphClass , cFeatureDefinition = cFeatureDefinition ) # == LEARNER CONFIGURATION === # setting the learner configuration, in a standard way # (from command line options, or from a JSON configuration file) dLearnerConfig = doer.getStandardLearnerConfig(options) # # force a balanced weighting # print("Forcing balanced weights") # dLearnerConfig['balanced'] = True # of course, you can put yours here instead. doer.setLearnerConfiguration(dLearnerConfig) # === GO!! === # act as per specified in the command line (--trn , --fold-run, ...) doer.standardDo(options) del doer
cFeatureDefinition=FeatureDefinition_PageXml_LogitExtractorV2) self.setNbClass( nbClass ) #so that we check if all classes are represented in the training set self.bsln_mdl = self.addBaseline_LogisticRegression( ) #use a LR model trained by GridSearch as baseline #=== END OF CONFIGURATION ============================================================= if __name__ == "__main__": version = "v.01" usage, description, parser = DU_Task_Factory.getStandardOptionsParser( sys.argv[0], version) # --- #parse the command line (options, args) = parser.parse_args() # --- try: sModelDir, sModelName = args except Exception as e: traceln("Specify a model folder and a model name!") _exit(usage, 1, e) doer = DU_GTBooks(sModelName, sModelDir, C=options.crf_C,
".//pc:TextLine" #how to find the nodes #, "./pc:TextEquiv") #how to get their text , ".//pc:Unicode") #how to get their text ) DU_GRAPH.addNodeType(nt) return DU_GRAPH if __name__ == "__main__": # import better_exceptions # better_exceptions.MAX_LENGTH = None # standard command line options for CRF- ECN- GAT-based methods usage, parser = DU_Task_Factory.getStandardOptionsParser(sys.argv[0]) traceln("VERSION: %s" % DU_Task_Factory.getVersion()) # --- #parse the command line (options, args) = parser.parse_args() try: sModelDir, sModelName = args except Exception as e: traceln("Specify a model folder and a model name!") DU_Task_Factory.exit(usage, 1, e) # === SETTING the graph type (and its node type) a,d the feature extraction pipe doer = DU_Task_Factory.getDoer(
#, "./pc:TextEquiv") #how to get their text , ".//pc:Unicode") #how to get their text ) DU_GRAPH.addNodeType(nt) return DU_GRAPH if __name__ == "__main__": # import better_exceptions # better_exceptions.MAX_LENGTH = None global options # standard command line options for CRF- ECN- GAT-based methods usage, parser = DU_Task_Factory.getStandardOptionsParser(sys.argv[0]) parser.add_option( "--separator", dest='bSeparator', action="store_true", default=False, help="Use the graphical spearators, if any, as edge features.") traceln("VERSION: %s" % DU_Task_Factory.getVersion()) # --- #parse the command line (options, args) = parser.parse_args() try: sModelDir, sModelName = args
def main(sys_argv_0, sLabelAttribute, cNodeType=NodeType_PageXml_type_woText): def getConfiguredGraphClass(_doer): """ In this class method, we must return a configured graph class """ DU_GRAPH = Graph_MultiSinglePageXml ntClass = cNodeType #lLabels = ['heading','paragraph','paragraph_left','paragraph_right','None'] lLabels = ['heading', 'paragraph', 'footnote', 'None'] #lLabels.append('IGNORE') nt = ntClass( sLabelAttribute #some short prefix because labels below are prefixed with it , lLabels # in conjugate, we accept all labels, andNone becomes "none" , [], False # unused , BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3) ) #we reduce overlap in this way ) nt.setLabelAttribute(sLabelAttribute) nt.setXpathExpr(( ".//pc:TextLine" #how to find the nodes #, "./pc:TextEquiv") #how to get their text , ".//pc:Unicode") #how to get their text ) DU_GRAPH.addNodeType(nt) return DU_GRAPH # standard command line options for CRF- ECN- GAT-based methods usage, parser = DU_Task_Factory.getStandardOptionsParser(sys_argv_0) parser.add_option( "--separator", dest='bSeparator', action="store_true", default=False, help="Use the graphical spearators, if any, as edge features.") parser.add_option( "--text", dest='bText', action="store_true", default=False, help="Use textual information if any, as node and edge features.") parser.add_option( "--edge_vh", "--edge_hv", dest='bShift', action="store_true", default=False, help="Shift edge feature by range depending on edge type.") parser.add_option( "--spm", dest='sSPModel', action="store", type="string", help= "Textual features are computed based on the given SentencePiece model. e.g. model/toto.model." ) traceln("VERSION: %s" % DU_Task_Factory.getVersion()) # --- #parse the command line (options, args) = parser.parse_args() try: sModelDir, sModelName = args except Exception as e: traceln("Specify a model folder and a model name!") DU_Task_Factory.exit(usage, 1, e) if options.sSPModel: if not (options.sSPModel.endswith(".model")): options.sSPModel = options.sSPModel + ".model" traceln( " - using SentencePiece model '%s' to create textual features" % options.sSPModel) # just checking things early... import sentencepiece as spm open(options.sSPModel).close() dFeatureConfig = {"sSPModel": options.sSPModel} cFeatureDefinition = Features_June19_Full_SPM options.bText = True elif options.bText: cFeatureDefinition = Features_GHENT_UNIGRAM dFeatureConfig = {} else: cFeatureDefinition = Features_GHENT dFeatureConfig = {} if options.bText: traceln(" - using textual data, if any") if options.bSeparator: traceln(" - using graphical separators, if any") if options.bShift: traceln(" - shift edge features by edge type") # if options.bText: # if options.bSeparator: # if options.bShift: # cFeatureDefinition = Features_June19_Full_Separator_Shift # else: # cFeatureDefinition = Features_June19_Full_Separator # else: # if options.bShift: # cFeatureDefinition = Features_June19_Full_Shift # else: # # cFeatureDefinition = Features_June19_Full # cFeatureDefinition = Features_BAR_Full # # else: # if options.bSeparator: # if options.bShift: # cFeatureDefinition = Features_June19_Simple_Separator_Shift # else: # cFeatureDefinition = Features_June19_Simple_Separator # else: # if options.bShift: # cFeatureDefinition = Features_June19_Simple_Shift # else: # cFeatureDefinition = Features_June19_Simple # === SETTING the graph type (and its node type) a,d the feature extraction pipe doer = DU_Task_Factory.getDoer( sModelDir, sModelName, options=options, fun_getConfiguredGraphClass=getConfiguredGraphClass, cFeatureDefinition=cFeatureDefinition, dFeatureConfig=dFeatureConfig) # == LEARNER CONFIGURATION === # setting the learner configuration, in a standard way # (from command line options, or from a JSON configuration file) dLearnerConfig = doer.getStandardLearnerConfig(options) # # force a balanced weighting # print("Forcing balanced weights") dLearnerConfig['balanced'] = True # of course, you can put yours here instead. doer.setLearnerConfiguration(dLearnerConfig) doer.setAdditionalDataProvider(getDataToPickle_for_table) # === CONJUGATE MODE === #doer.setConjugateMode() # === GO!! === # act as per specified in the command line (--trn , --fold-run, ...) doer.standardDo(options) del doer
) nt.setLabelAttribute("type") nt.setXpathExpr((".//pc:TextRegion" , "") ) DU_GRAPH.addNodeType(nt) return DU_GRAPH if __name__ == "__main__": # import better_exceptions # better_exceptions.MAX_LENGTH = None # standard command line options for CRF- ECN- GAT-based methods usage, parser = DU_Task_Factory.getStandardOptionsParser(sys.argv[0]) parser.add_option("--TextRegion" , dest='bTextRegion' , action="store_true" , default=False, help="Tag TextRegion, instead of TextLine") parser.add_option("--no_text" , dest='bNoText' , action="store_true" , default=False, help="Do not use text, otherwise use utext unigrams") traceln("VERSION: %s" % DU_Task_Factory.getVersion()) # --- #parse the command line (options, args) = parser.parse_args() bTextRegion = options.bTextRegion bText = not(options.bNoText) traceln("bTextRegion=%s bText=%s" % (bTextRegion, bText))