def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
lLabels = ['B', 'O', 'I']
lIgnoredLabels = None
"""
if you play with a toy collection, which does not have all expected classes, you can reduce those.
"""
lActuallySeen = None
if lActuallySeen:
print("REDUCING THE CLASSES TO THOSE SEEN IN TRAINING")
lIgnoredLabels = [
lLabels[i] for i in range(len(lLabels))
if i not in lActuallySeen
]
lLabels = [lLabels[i] for i in lActuallySeen]
print(len(lLabels), lLabels)
print(len(lIgnoredLabels), lIgnoredLabels)
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiSinglePageXml
nt = NodeType_PageXml_type_woText(
"abp" #some short prefix because labels below are prefixed with it
,
lLabels,
lIgnoredLabels,
False #no label means OTHER
,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
nt.setLabelAttribute("DU_row")
# ntA = NodeType_PageXml_type_woText("abp" #some short prefix because labels below are prefixed with it
# , lLabels
# , lIgnoredLabels
# , False #no label means OTHER
# )
nt.setXpathExpr((
".//pc:TextLine" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
# ntA.setXpathExpr( (".//pc:TextLine | .//pc:TextRegion" #how to find the nodes
# , "./pc:TextEquiv") #how to get their text
# )
DU_GRAPH.addNodeType(nt)
return DU_GRAPH
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
# Textline labels
# Begin Inside End Single Other
lLabels_SIO_row = ['S', 'I', 'O']
# Cut lines:
# Border Ignore Separator Outside
lLabels_SIO_Cut = ['S', 'I', 'O']
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = GraphCut_H
DU_GRAPH.iBlockVisibility = cls.iBlockVisibility
DU_GRAPH.iLineVisibility = cls.iLineVisibility
# ROW
ntR = NodeType_BIESO_to_SIO(
"row",
lLabels_SIO_row,
None,
False,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)))
ntR.setLabelAttribute("DU_row")
ntR.setXpathExpr((
".//pc:TextLine" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntR)
# HEADER
ntCutH = NodeType_PageXml_type_woText(
"sepH",
lLabels_SIO_Cut,
None,
False,
None # equiv. to: BBoxDeltaFun=lambda _: 0
)
ntCutH.setLabelAttribute("type")
ntCutH.setXpathExpr((
'.//pc:CutSeparator[@orient="0"]' #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntCutH)
DU_GRAPH.setClassicNodeTypeList([ntR])
return DU_GRAPH
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
lLabels_COLUMN_HEADER = [
'CH',
'D',
'O',
]
# """
# if you play with a toy collection, which does not have all expected classes, you can reduce those.
# """
#
# lActuallySeen = None
# if lActuallySeen:
# print( "REDUCING THE CLASSES TO THOSE SEEN IN TRAINING")
# lIgnoredLabels = [lLabelsR[i] for i in range(len(lLabelsR)) if i not in lActuallySeen]
# lLabels = [lLabelsR[i] for i in lActuallySeen ]
# print( len(lLabelsR) , lLabelsR)
# print( len(lIgnoredLabels) , lIgnoredLabels)
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiContinousPageXml
# HEADER
ntH = NodeType_PageXml_type_woText(
"hdr",
lLabels_COLUMN_HEADER,
None,
False #no label means OTHER
,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
ntH.setLabelAttribute("DU_header")
ntH.setXpathExpr((
".//pc:TextLine" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntH)
return DU_GRAPH
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
lLabelsBIEOS_R = ['B', 'I', 'E', 'S', 'O'] #O?
#lLabelsSM_C = ['M', 'S', 'O'] # single cell, multicells
# lLabels_OI = ['O','I'] # inside/outside a table
# lLabels_SPAN = ['rspan','cspan','nospan','OTHER']
#lLabels_COLUMN_HEADER = ['CH', 'D', 'O',]
# """
# if you play with a toy collection, which does not have all expected classes, you can reduce those.
# """
#
# lActuallySeen = None
# if lActuallySeen:
# print( "REDUCING THE CLASSES TO THOSE SEEN IN TRAINING")
# lIgnoredLabels = [lLabelsR[i] for i in range(len(lLabelsR)) if i not in lActuallySeen]
# lLabels = [lLabelsR[i] for i in lActuallySeen ]
# print( len(lLabelsR) , lLabelsR)
# print( len(lIgnoredLabels) , lIgnoredLabels)
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiContinousPageXml
# ROW
ntR = NodeType_PageXml_type_woText(
"row",
lLabelsBIEOS_R,
None,
False #no label means OTHER
,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
ntR.setLabelAttribute("DU_row")
ntR.setXpathExpr((
".//pc:TextLine" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntR)
return DU_GRAPH
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiPageXml
lLabels1 = [
'heading', 'header', 'page-number', 'resolution-number',
'resolution-marginalia', 'resolution-paragraph', 'other'
]
#the converter changed to other unlabelled TextRegions or 'marginalia' TRs
lIgnoredLabels1 = None
"""
if you play with a toy collection, which does not have all expected classes, you can reduce those.
"""
# lActuallySeen = None
# if lActuallySeen:
# print( "REDUCING THE CLASSES TO THOSE SEEN IN TRAINING")
# lIgnoredLabels = [lLabels[i] for i in range(len(lLabels)) if i not in lActuallySeen]
# lLabels = [lLabels[i] for i in lActuallySeen ]
# print( len(lLabels) , lLabels)
# print( len(lIgnoredLabels) , lIgnoredLabels)
nt1 = NodeType_PageXml_type_woText(
"sem" #some short prefix because labels below are prefixed with it
,
lLabels1,
lIgnoredLabels1,
False #no label means OTHER
,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
nt1.setLabelAttribute("DU_sem")
nt1.setXpathExpr((
".//pc:TextRegion" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(nt1)
return DU_GRAPH
lLabels = [lLabels[i] for i in lActuallySeen ]
print (len(lLabels) , lLabels)
print (len(lIgnoredLabels) , lIgnoredLabels)
nbClass = len(lLabels) + 1 #because the ignored labels will become OTHER
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiPageXml
# nt = NodeType_PageXml_type_woText("abp" #some short prefix because labels below are prefixed with it
# , lLabels
# , lIgnoredLabels
# , False #no label means OTHER
# )
ntA = NodeType_PageXml_type_woText("abp" #some short prefix because labels below are prefixed with it
, lLabels
, lIgnoredLabels
, False #no label means OTHER
)
# nt.setXpathExpr( (".//pc:TextLine" #how to find the nodes
# , "./pc:TextEquiv") #how to get their text
# )
ntA.setXpathExpr( (".//pc:TextLine | .//pc:TextRegion | .//pc:SeparatorRegion" #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
# ===============================================================================================================
def __init__(self,
sModelName,
sModelDir,
sComment=None,
C=None,
tol=None,
njobs=None,
max_iter=None,
inference_cache=None):
# ===============================================================================================================
lLabels = ['RB', 'RI', 'RE', 'RS', 'RO']
lIgnoredLabels = None
nbClass = len(lLabels)
"""
if you play with a toy collection, which does not have all expected classes, you can reduce those.
"""
lActuallySeen = None
if lActuallySeen:
print "REDUCING THE CLASSES TO THOSE SEEN IN TRAINING"
lIgnoredLabels = [
lLabels[i] for i in range(len(lLabels))
if i not in lActuallySeen
]
lLabels = [lLabels[i] for i in lActuallySeen]
print len(lLabels), lLabels
print len(lIgnoredLabels), lIgnoredLabels
nbClass = len(
lLabels) + 1 #because the ignored labels will become OTHER
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiSinglePageXml
nt = NodeType_PageXml_type_woText(
"abp" #some short prefix because labels below are prefixed with it
,
lLabels,
lIgnoredLabels,
False #no label means OTHER
,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
# ntA = NodeType_PageXml_type_woText("abp" #some short prefix because labels below are prefixed with it
# , lLabels
# , lIgnoredLabels
# , False #no label means OTHER
# )
nt.setXpathExpr((
".//pc:TextLine" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
# ntA.setXpathExpr( (".//pc:TextLine | .//pc:TextRegion" #how to find the nodes
# , "./pc:TextEquiv") #how to get their text
# )
DU_GRAPH.addNodeType(nt)
# ===============================================================================================================
DU_CRF_Task.__init__(
self,
sModelName,
sModelDir,
DU_GRAPH,
dFeatureConfig={},
dLearnerConfig={
'C':
.1 if C is None else C,
'njobs':
8 if njobs is None else njobs,
'inference_cache':
50 if inference_cache is None else inference_cache
#, 'tol' : .1
,
'tol':
.05 if tol is None else tol,
'save_every':
50 #save every 50 iterations,for warm start
,
'max_iter':
1000 if max_iter is None else max_iter
},
sComment=sComment
#,cFeatureDefinition=FeatureDefinition_PageXml_StandardOnes_noText
,
cFeatureDefinition=FeatureDefinition_PageXml_NoNodeFeat_v3)
#self.setNbClass(3) #so that we check if all classes are represented in the training set
self.bsln_mdl = self.addBaseline_LogisticRegression(
) #use a LR model trained by GridSearch as baseline
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
# Textline labels
# Begin Inside End Single Other
lLabels_BIESO = ['B', 'I', 'E', 'S', 'O']
# Grid lines:
# Border Ignore Separator Outside
lLabels_BISO_Grid = ['B', 'I', 'S', 'O']
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = GraphGrid_H
DU_GRAPH.iGridStep_H = cls.iGridStep_H
DU_GRAPH.iGridStep_V = cls.iGridStep_V
DU_GRAPH.iGridVisibility = cls.iGridVisibility
DU_GRAPH.iBlockVisibility = cls.iBlockVisibility
# ROW
ntR = NodeType_PageXml_type_woText(
"row",
lLabels_BIESO,
None,
False,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
ntR.setLabelAttribute("DU_row")
ntR.setXpathExpr((
".//pc:TextLine" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntR)
# HEADER
ntGH = NodeType_PageXml_type_woText(
"gh",
lLabels_BISO_Grid,
None,
False,
None # equiv. to: BBoxDeltaFun=lambda _: 0
)
ntGH.setLabelAttribute("type")
ntGH.setXpathExpr((
'.//pc:GridSeparator[@orient="0"]' #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntGH)
DU_GRAPH.setClassicNodeTypeList([ntR])
return DU_GRAPH
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
# Textline labels
# Begin Inside End Single Other
lLabels_BIESO = ['B', 'I', 'E', 'S', 'O']
lLabels_COLUMN_HEADER = ['CH', 'D', 'O',]
# Cut lines:
# Border Ignore Separator Outside
lLabels_SIO_Cut = ['S', 'I', 'O']
#DEFINING THE CLASS OF GRAPH WE USE
# this is an ad-hoc class where type1 and type2 are factorial, while type3 is artificial object
DU_GRAPH = AdHocFactorialGraphCut_H
DU_GRAPH.iBlockVisibility = cls.iBlockVisibility
DU_GRAPH.iLineVisibility = cls.iLineVisibility
# ROW
ntR = NodeType_PageXml_type_woText("row"
, lLabels_BIESO
, None
, False
, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3))
)
ntR.setLabelAttribute("DU_row")
ntR.setXpathExpr( (".//pc:TextLine" #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntR)
# HEADER
ntH = NodeType_PageXml_type_woText("hdr"
, lLabels_COLUMN_HEADER
, None
, False #no label means OTHER
, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3)) #we reduce overlap in this way
)
ntH.setLabelAttribute("DU_header")
ntH.setXpathExpr( (".//pc:TextLine" #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntH)
# HEADER
ntCutH = NodeType_PageXml_type_woText("sepH"
, lLabels_SIO_Cut
, None
, False
, None # equiv. to: BBoxDeltaFun=lambda _: 0
)
ntCutH.setLabelAttribute("type")
ntCutH.setXpathExpr( ('.//pc:CutSeparator[@orient="0"]' #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntCutH)
# The nodes of this type (called "classic") are directly extracted from the XML
# the other types of nodes are computed
DU_GRAPH.setClassicNodeTypeList([ntR])
DU_GRAPH.setSpecialNodeTypeList([ntCutH])
DU_GRAPH.setFactoredClassicalType(ntR, ntH) # make ntH a factorial of ntR
return DU_GRAPH
class DU_BAR_sgm(DU_CRF_Task):
"""
We will do a typed CRF model for a DU task
, with the below labels
"""
sLabeledXmlFilenamePattern = "*.du_mpxml"
# ===============================================================================================================
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiContinousPageXml
#lLabels2 = ['heigh', 'ho', 'other']
#lLabels2 = ['heigh', 'ho']
lLabels2 = ['B', 'I', 'E'] #we never see any S... , 'S']
# Some TextRegion have no segmentation label at all, and were labelled'other' by the converter
lIgnoredLabels2 = None
# """
# if you play with a toy collection, which does not have all expected classes, you can reduce those.
# """
#
# lActuallySeen = None
# if lActuallySeen:
# print "REDUCING THE CLASSES TO THOSE SEEN IN TRAINING"
# lIgnoredLabels = [lLabels[i] for i in range(len(lLabels)) if i not in lActuallySeen]
# lLabels = [lLabels[i] for i in lActuallySeen ]
# print len(lLabels) , lLabels
# print len(lIgnoredLabels) , lIgnoredLabels
# nbClass = len(lLabels) + 1 #because the ignored labels will become OTHER
nt2 = NodeType_PageXml_type_woText(
"sgm" #some short prefix because labels below are prefixed with it
,
lLabels2,
lIgnoredLabels2,
False #no label means OTHER
,
BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v / 3)
) #we reduce overlap in this way
)
nt2.setLabelAttribute("DU_sgm")
nt2.setXpathExpr((
".//pc:TextRegion" #how to find the nodes
,
"./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(nt2)
#=== CONFIGURATION ====================================================================
def __init__(self,
sModelName,
sModelDir,
sComment=None,
C=None,
tol=None,
njobs=None,
max_iter=None,
inference_cache=None):
DU_CRF_Task.__init__(
self,
sModelName,
sModelDir,
self.DU_GRAPH,
dLearnerConfig={
'C':
.1 if C is None else C,
'njobs':
8 if njobs is None else njobs,
'inference_cache':
50 if inference_cache is None else inference_cache
#, 'tol' : .1
,
'tol':
.05 if tol is None else tol,
'save_every':
50 #save every 50 iterations,for warm start
,
'max_iter':
1000 if max_iter is None else max_iter
},
sComment=sComment,
cFeatureDefinition=FeatureDefinition_PageXml_StandardOnes_noText
# , cFeatureDefinition=FeatureDefinition_T_PageXml_StandardOnes_noText
# , dFeatureConfig = {
# #config for the extractor of nodes of each type
# "text": None,
# "sprtr": None,
# #config for the extractor of edges of each type
# "text_text": None,
# "text_sprtr": None,
# "sprtr_text": None,
# "sprtr_sprtr": None
# }
)
traceln("- classes: ", self.DU_GRAPH.getLabelNameList())
self.bsln_mdl = self.addBaseline_LogisticRegression(
) #use a LR model trained by GridSearch as baseline
#=== END OF CONFIGURATION =============================================================
def predict(self, lsColDir, sDocId):
"""
Return the list of produced files
"""
# self.sXmlFilenamePattern = "*.a_mpxml"
return DU_CRF_Task.predict(self, lsColDir, sDocId)
class DU_BAR_sem_sgm(DU_FactorialCRF_Task):
"""
We will do a Factorial CRF model using the Multitype CRF
, with the below labels
"""
sLabeledXmlFilenamePattern = "*.du_mpxml"
# ===============================================================================================================
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = FactorialGraph_MultiContinuousPageXml
#---------------------------------------------
lLabels1 = ['heading', 'header', 'page-number', 'resolution-number', 'resolution-marginalia', 'resolution-paragraph', 'other']
nt1 = NodeType_PageXml_type_woText("sem" #some short prefix because labels below are prefixed with it
, lLabels1
, None #keep this to None, unless you know very well what you do. (FactorialCRF!)
, False #no label means OTHER
, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3)) #we reduce overlap in this way
)
nt1.setLabelAttribute("DU_sem")
nt1.setXpathExpr( (".//pc:TextRegion" #how to find the nodes, MUST be same as for other node type!! (FactorialCRF!)
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(nt1)
#---------------------------------------------
#lLabels2 = ['heigh', 'ho', 'other']
#lLabels2 = ['heigh', 'ho']
lLabels2 = ['B', 'I', 'E'] #we never see any S... , 'S']
lLabels2 = ['B', 'I', 'E', 'S', 'O'] #we never see any S... , 'S']
nt2 = NodeType_PageXml_type_woText("sgm" #some short prefix because labels below are prefixed with it
, lLabels2
, None #keep this to None, unless you know very well what you do. (FactorialCRF!)
, False #no label means OTHER
, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3)) #we reduce overlap in this way
)
nt2.setLabelAttribute("DU_sgm")
nt2.setXpathExpr( (".//pc:TextRegion" #how to find the nodes, MUST be same as for other node type!! (FactorialCRF!)
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(nt2)
#=== CONFIGURATION ====================================================================
def __init__(self, sModelName, sModelDir, sComment=None, C=None, tol=None, njobs=None, max_iter=None, inference_cache=None):
# #edge feature extractor config is a bit teddious...
# dFeatureConfig = { lbl:None for lbl in self.lLabels1+self.lLabels2 }
# for lbl1 in self.lLabels1:
# for lbl2 in self.lLabels2:
# dFeatureConfig["%s_%s"%(lbl1, lbl2)] = None
DU_FactorialCRF_Task.__init__(self
, sModelName, sModelDir
, self.DU_GRAPH
, dLearnerConfig = {
'C' : .1 if C is None else C
, 'njobs' : 8 if njobs is None else njobs
, 'inference_cache' : 50 if inference_cache is None else inference_cache
#, 'tol' : .1
, 'tol' : .05 if tol is None else tol
, 'save_every' : 50 #save every 50 iterations,for warm start
, 'max_iter' : 1000 if max_iter is None else max_iter
}
, sComment=sComment
, cFeatureDefinition=FeatureDefinition_PageXml_StandardOnes_noText
# , cFeatureDefinition=FeatureDefinition_T_PageXml_StandardOnes_noText
# {
# #config for the extractor of nodes of each type
# "text": None,
# "sprtr": None,
# #config for the extractor of edges of each type
# "text_text": None,
# "text_sprtr": None,
# "sprtr_text": None,
# "sprtr_sprtr": None
# }
)
traceln("- classes: ", self.DU_GRAPH.getLabelNameList())
self.bsln_mdl = self.addBaseline_LogisticRegression() #use a LR model trained by GridSearch as baseline
#=== END OF CONFIGURATION =============================================================
def predict(self, lsColDir,sDocId):
"""
Return the list of produced files
"""
# self.sXmlFilenamePattern = "*.a_mpxml"
return DU_FactorialCRF_Task.predict(self, lsColDir,sDocId)
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
# Textline labels
# Begin Inside End Single Other
lLabels_BIESO = ['B', 'I', 'E', 'S', 'O']
# Grid lines:
# Border Ignore Separator Outside
lLabels_BISO_Grid = ['B', 'I', 'S', 'O']
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = GraphGrid_H
DU_GRAPH.iGridStep_H = cls.iGridStep_H
DU_GRAPH.iGridStep_V = cls.iGridStep_V
DU_GRAPH.iGridVisibility = cls.iGridVisibility
DU_GRAPH.iBlockVisibility = cls.iBlockVisibility
# ROW
ntR = NodeType_PageXml_type_woText("row"
, lLabels_BIESO
, None
, False
#HISTORICAL FUNCTION IS (idiotic I think...):
#, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3))
, BBoxDeltaFun=lambda v: v / 5.0, #keep 2/3rd of the box
# we reduce overlap in this way
#this function returns the amount by which each border of
# a bounding box is "shifted toward its centre"...
# w,h = x2-x1, y2-y1
# dx = self.BBoxDeltaFun(w)
# dy = self.BBoxDeltaFun(h)
# x1,y1, x2,y2 = [ int(round(v)) for v in [x1+dx,y1+dy, x2-dx,y2-dy] ]
)
ntR.setLabelAttribute("DU_row")
ntR.setXpathExpr( (".//pc:TextLine" #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntR)
# HEADER
ntGH = NodeType_PageXml_type_woText("gh"
, lLabels_BISO_Grid
, None
, False
, None # equiv. to: BBoxDeltaFun=lambda _: 0
)
ntGH.setLabelAttribute("type")
ntGH.setXpathExpr( ('.//pc:GridSeparator[@orient="0"]' #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(ntGH)
DU_GRAPH.setClassicNodeTypeList( [ntR ])
return DU_GRAPH
class DU_ABPTable_TypedCRF(DU_CRF_Task):
"""
We will do a typed CRF model for a DU task
, with the below labels
"""
sXmlFilenamePattern = "*.mpxml"
sLabeledXmlFilenamePattern = "*.mpxml"
sLabeledXmlFilenameEXT = ".mpxml"
#=== CONFIGURATION ====================================================================
@classmethod
def getConfiguredGraphClass(cls):
"""
In this class method, we must return a configured graph class
"""
# ===============================================================================================================
#DEFINING THE CLASS OF GRAPH WE USE
DU_GRAPH = Graph_MultiSinglePageXml
lLabels1 = ['RB', 'RI', 'RE', 'RS','RO']
lIgnoredLabels1 = None
# """
# if you play with a toy collection, which does not have all expected classes, you can reduce those.
# """
#
# lActuallySeen = None
# if lActuallySeen:
# print "REDUCING THE CLASSES TO THOSE SEEN IN TRAINING"
# lIgnoredLabels = [lLabels[i] for i in range(len(lLabels)) if i not in lActuallySeen]
# lLabels = [lLabels[i] for i in lActuallySeen ]
# print len(lLabels) , lLabels
# print len(lIgnoredLabels) , lIgnoredLabels
# nbClass = len(lLabels) + 1 #because the ignored labels will become OTHER
nt1 = NodeType_PageXml_type_woText("text" #some short prefix because labels below are prefixed with it
, lLabels1
, lIgnoredLabels1
, False #no label means OTHER
, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3)) #we reduce overlap in this way
)
nt1.setXpathExpr( (".//pc:TextLine" #how to find the nodes
, "./pc:TextEquiv") #how to get their text
)
DU_GRAPH.addNodeType(nt1)
nt2 = NodeType_PageXml_type_woText("sprtr" #some short prefix because labels below are prefixed with it
, ['SI', 'SO']
, None
, False #no label means OTHER
, BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3)) #we reduce overlap in this way
)
nt2.setXpathExpr( (".//pc:SeparatorRegion" #how to find the nodes
, "./pc:TextEquiv") #how to get their text (no text in fact)
)
DU_GRAPH.addNodeType(nt2)
#=== CONFIGURATION ====================================================================
def __init__(self, sModelName, sModelDir, sComment=None, C=None, tol=None, njobs=None, max_iter=None, inference_cache=None):
#another way to specify the graph class
# defining a getConfiguredGraphClass is preferred
self.configureGraphClass(self.DU_GRAPH)
DU_CRF_Task.__init__(self
, sModelName, sModelDir
, dLearnerConfig = {
'C' : .1 if C is None else C
, 'njobs' : 8 if njobs is None else njobs
, 'inference_cache' : 50 if inference_cache is None else inference_cache
#, 'tol' : .1
, 'tol' : .05 if tol is None else tol
, 'save_every' : 50 #save every 50 iterations,for warm start
, 'max_iter' : 1000 if max_iter is None else max_iter
}
, sComment=sComment
, cFeatureDefinition=FeatureDefinition_T_PageXml_StandardOnes_noText_v3
, dFeatureConfig = {
#config for the extractor of nodes of each type
"text": None,
"sprtr": None,
#config for the extractor of edges of each type
"text_text": None,
"text_sprtr": None,
"sprtr_text": None,
"sprtr_sprtr": None
}
)
traceln("- classes: ", self.DU_GRAPH.getLabelNameList())
self.bsln_mdl = self.addBaseline_LogisticRegression() #use a LR model trained by GridSearch as baseline
#=== END OF CONFIGURATION =============================================================
def predict(self, lsColDir,sDocId):
"""
Return the list of produced files
"""
# self.sXmlFilenamePattern = "*.a_mpxml"
return DU_CRF_Task.predict(self, lsColDir,sDocId)
