def TrainUsingCRF(xmls, preprocessor, trainer):
    CRFImpl = CRF()
    annotatedxmllist = list()
    for xmlname in xmls:
        fontdict = preprocessor.getFontDictionary(ET.parse("../TrainingData/xmls/cs/" + xmlname + ".xml")) #list(pages), pages -> list(cols), col -> list(<Sparse/NonSparse, tag>)
        annotatedxml = trainer.readAnnotatedXml('../TrainingData/annotated/' + xmlname + "_annotated")
        annotatedxmllist.append([annotatedxml, fontdict])
    CRFImpl.domaintrain(annotatedxmllist)
    f = open("TrainedWeightsCRF", 'w')
    for weight in CRFImpl.trainedweights:
        f.write(str(weight) + "\n")
    
    f.close()
def TestUsingCRF(predictxmlname, location):
    CRF = getModelwithTrainedWeights()
    fontdict = preprocessor.getFontDictionary(ET.parse(location + predictxmlname + ".xml"))                  
    preprocessedxml = preprocessor.preprocessxml(location + predictxmlname + ".xml") #list(pages), pages -> list(cols), col -> list(<Sparse/NonSparse, tag>)
    alltables = list()
    for page in preprocessedxml:
        for col in page:
            if(len(col) < 2):
                    continue
            for lineno in xrange(len(col)):
                col[lineno].append(lineno)
            predicted = CRF.predict(col, fontdict)
            for r in predicted:
#                if(r[0] == SparseType.OTHERSPARSE):
                    print r[1].text + " *** Line no *** " + str(r[2]) + " --  " + str(r[0])
            data = postprocessor.findTables(predicted)
            tables = data
            if(len(tables) == 0):
                continue
            for t in tables:
                alltables.append(t)
    
    for table in alltables:
        print "============================================="
        for row in table:
            print row[1].text + " " + str(row[0]) 
def TestUsingCRF(predictxmlname, location, TDsvm=None):
    CRF = getModelwithTrainedWeights()
    fontdict = preprocessor.getFontDictionary(
        ET.parse(location + predictxmlname + ".xml"))
    preprocessedxml = preprocessor.preprocessxml(
        location + predictxmlname + ".xml"
    )  #list(pages), pages -> list(cols), col -> list(<Sparse/NonSparse, tag>)

    alltables = list()
    errorcount = 0
    sparseerror = 0
    ntlafterpostproc = 0
    for page in preprocessedxml:
        for col in page:
            if (len(col) < 2):
                continue
            for tup in col:
                if (tup[1].text is None or tup[1].text.strip() == ''):
                    col.remove(tup)
            for lineno in xrange(len(col)):
                col[lineno].append(lineno)

            result = CRF.predict(col, fontdict)
            predicted = result[0]
            errorcount += result[1]
            sparseerror += result[2]
            #            for r in predicted:
            #                if(r[0] == SparseType.OTHERSPARSE):
            #                    print r[1].text.encode('ascii','ignore') + " *** Line no *** " + str(r[2])
            data = postprocessor.findTables(predicted)
            tables = data
            if (len(tables) == 0):
                continue
            for t in tables:
                alltables.append(t)

    if TDsvm is None:
        for table in alltables:
            print "============================================="
            for row in table:
                if (int(row[0]) == SparseType.NONTABLELINE):
                    ntlafterpostproc += 1
                print row[1].text.encode('ascii', 'ignore')
            print "=============================================="

    else:
        for t in alltables:
            predicted = TDsvm.domainpredictforTableDecomposition(t)
            print "=============================================="
            for r in predicted[0]:
                if (r[0] == SparseType.HEADER):
                    print r[1].text + "  ---> HEADER "
                else:
                    print r[1].text + "  ---> DATA "
            print "=============================================="

    return [errorcount, sparseerror, ntlafterpostproc]
def TrainUsingCRF(xmls, preprocessor, trainer, xmlloc, annotatedxmlloc):
    CRFImpl = CRF()
    annotatedxmllist = list()
    for xmlname in xmls:
        fontdict = preprocessor.getFontDictionary(
            ET.parse(xmlloc + xmlname + ".xml")
        )  #list(pages), pages -> list(cols), col -> list(<Sparse/NonSparse, tag>)
        annotatedxml = trainer.readAnnotatedXml(annotatedxmlloc + xmlname +
                                                "_annotated")
        annotatedxmllist.append([annotatedxml, fontdict])

    CRFImpl.domaintrain(annotatedxmllist)
    print CRFImpl.trainedweights
    f = open("TrainedWeightsCRF", 'w')
    for weight in CRFImpl.trainedweights:
        f.write(str(weight) + "\n")

    f.close()
def TestUsingCRF(predictxmlname, location, TDsvm = None):
    CRF = getModelwithTrainedWeights()
    fontdict = preprocessor.getFontDictionary(ET.parse(location + predictxmlname + ".xml"))                  
    preprocessedxml = preprocessor.preprocessxml(location + predictxmlname + ".xml") #list(pages), pages -> list(cols), col -> list(<Sparse/NonSparse, tag>)
    
    alltables = list()
    errorcount = 0
    sparseerror = 0
    ntlafterpostproc = 0
    for page in preprocessedxml:
        for col in page:
            if(len(col) < 2):
                    continue
            for tup in col:
                if(tup[1].text is None or tup[1].text.strip() == ''):
                    col.remove(tup)
            for lineno in xrange(len(col)):
                col[lineno].append(lineno)
            
            result = CRF.predict(col, fontdict)
            predicted = result[0]
            errorcount += result[1]
            sparseerror += result[2]
#            for r in predicted:
#                if(r[0] == SparseType.OTHERSPARSE):
#                    print r[1].text.encode('ascii','ignore') + " *** Line no *** " + str(r[2])
            data = postprocessor.findTables(predicted)
            tables = data
            if(len(tables) == 0):
                continue
            for t in tables:
                alltables.append(t)
                
    if TDsvm is None:
        for table in alltables:
            print "============================================="
            for row in table:
                if(int(row[0]) == SparseType.NONTABLELINE):
                    ntlafterpostproc += 1
                print row[1].text.encode('ascii','ignore') 
            print "=============================================="
        
    else:
        for t in alltables:
            predicted = TDsvm.domainpredictforTableDecomposition(t)
            print "=============================================="
            for r in predicted[0]:
                if(r[0] == SparseType.HEADER):
                    print r[1].text + "  ---> HEADER "
                else:
                    print r[1].text + "  ---> DATA "
            print "=============================================="
            
    return [errorcount, sparseerror, ntlafterpostproc]
def getModelwithTrainedWeights(isCRF=True):
    trainedweights = list()
    if (isCRF):
        f = open("TrainedWeightsCRF", "r")
        for weight in f:
            trainedweights.append(float(weight))

        f.close()
        CRFImpl = CRF(trainedweights)
        return CRFImpl
    else:
        f = open("TrainedWeightsLR", "r")
        for weight in f:
            trainedweights.append(float(weight))

        f.close()
        LR = LogisticRegressor(trainedweights)
        return LR