__author__ = 'nparslow'

from compareCorrectedCorpus import getFinalTokenFormsAndTreesAndWeight

import calcPLex

print "loading freq info"
lemmacat2freqrank = calcPLex.loadLemmaCat2freqrank()


# testxmls = ["analysed_SpellChecker/entry_81/0/0/0/entry_81.E1.dep.xml"

testxmls = [
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E1.dep.xml",
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E5.dep.xml",
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E2.dep.xml",
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E6.dep.xml",
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E3.dep.xml",
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E7.dep.xml",
    "/home/nparslow/Documents/AutoCorrige/Corpora/analysed_SpellChecker/entry_350/0/0/0/entry_350.E4.dep.xml",
    ]

wordforms = []
for testxml in testxmls:
    print "loading xml", testxml
    tok2finalforms, tok2lemmacats, verb2info, trees, (weight, maxweight) = getFinalTokenFormsAndTreesAndWeight(testxml)
    # each tok2lemmacat goes to a list to 2-tuples
    # print tok2lemmacats.values()
    for lemmacats in tok2lemmacats.values():
        wordforms.extend([x[0] + u"_" + x[1] for x in lemmacats])
def main(argv):
    # todo add a debug variable and optionfilename variable
    optionsfilename = "settings/optionsfile.txt"
    try:
        opts, args = getopt.getopt(argv,"hi:", ["in_options_file="])
    except getopt.GetoptError:
        print 'textExtractor.py -i <input options file>'
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print 'textExtractor.py -i <input options file>'
            sys.exit()
        elif opt in ("-i", "--in_options_file"):
            optionsfilename = arg 

    globalparams, variableparams = optionsFileReader.readOptionsFile(optionsfilename)
    filenames = allCorpusFiles( globalparams["origtextdir"] )
    print filenames
    # todo for the moment we just take the first element for these
    #meltDir = globalparams["melteddir"][0]
    #ddagDir = globalparams["ddageddir"][0]
    #frmgDir = globalparams["frmgeddir"][0]
    filenameResources = {
        "filename": None,
        "melteddir" : globalparams["melteddir"][0],
        "ddageddir" : globalparams["ddageddir"][0],
        "frmgeddir" : globalparams["frmgeddir"][0],
    }

    print globalparams
    outarffdir = globalparams["outdir"][0]
    corpusName = globalparams["corpusName"][0]
    headerInfo = globalparams["headerInfo"][0]
    lexiquefile = globalparams["lexiqueDict"][0]
    freq2ranksfile = globalparams["freq2ranks"][0]

    gensimModelFile = globalparams["word2vecmodel"][0]

    variableTypes = set([x[0] for x in variableparams])

    lemmacat2freqrank = {}
    if "PLex" in variableTypes or "S" in variableTypes or "altS" in variableTypes or "LFP" in variableTypes:
        print "loading freq info"
        lemmacat2freqrank = calcPLex.loadLemmaCat2freqrank(freq2ranksfile)

    word2vecModel = None
    if "w2vct" in variableTypes:
        print "loading word2vec model"
        word2vecModel = gensim.models.Word2Vec.load(gensimModelFile)

    nGramDict, nmoGramDict, totalcounts = {}, {}, 1000000000000000
    if "bigramLogProb" in variableTypes:
        print "loading bi-gram model"
        ngramModelFile = globalparams["bigrammodel"][0]
        nGramDict, nmoGramDict, totalcounts = nGramModel.getNgramDicts(ngramModelFile)

    lexiqueDict = {}
    if "syllablesPerWord" in variableTypes:
        loadLexiqueToDict(lexiquefile, lexiqueDict)

    # create a list of variables for the arff file:
    variables = [ ("filename", "string")] + \
        [ (variableAndParamsToString(vname, params), "numeric") for vname,params in variableparams] + \
        [ ("level", "numeric")] # note that level must go last!

    for x in variables:
        print x

    #filenames = ["/home/nparslow/Documents/AutoCorrige/Corpora/CORPUS_CEFLE/C/Catja.txt"]
    #filenames = ["/home/nparslow/Documents/AutoCorrige/Corpora/CORPUS_CEFLE/A/Arvid.txt"]
    #filenames = ["/home/nparslow/Documents/AutoCorrige/Corpora/CORPUS_CEFLE/A/Amie4.txt"]
    #filenames = ["/home/nparslow/Documents/AutoCorrige/Corpora/CORPUS_CEFLE/B/Bror2.txt"]
    filenames = ["/home/nparslow/Documents/AutoCorrige/Corpora/CORPUS_CEFLE/C/Caroline.txt"]
    filenames = ["/home/nparslow/Documents/AutoCorrige/Corpora/CORPUS_CEFLE/E/Eddy.txt"] # just to test an E
    filenames = ["/home/pinot/alpage/nparslow/Documents/Corpora/CEFLE/E/Eddy.txt"] # just to test an E


    resources = {
        "lemmacat2freqrank": lemmacat2freqrank,
        "word2vecModel": word2vecModel,
        "nGramDict": nGramDict,
        "nmoGramDict": nmoGramDict,
        "nGramCounts": totalcounts,
        "lexiqueDict": lexiqueDict
    }

    outputRows =[]


    allobservedtrees = set([])
    treespertext = []
    for filename in filenames:

        # we change the resouce filename with each round:
        filenameResources["filename"] = filename
        baseFileName = os.path.basename(filename)
        baseFileName, extension = os.path.splitext(baseFileName)
        filenameResources["ddagfile"] = os.path.join(filenameResources["ddageddir"],
                                                         os.path.basename(baseFileName) + ".ddag")

        print
        print filename
        #fname = os.path.join(baseDir, filename)
        #if "CEFLE" in baseDir:
        #    fname = os.path.join(baseDir, filename[0], filename)

        #text = getDocumentProperties(frmgDir, meltDir, ddagDir, fname, word2vecModel,
        #                             (nGramDict, nmoGramDict, totalcounts), debug=False)
        #text = getDocumentProperties(filenameResources, variables, word2vecModel,
        #                             (nGramDict, nmoGramDict, totalcounts), debug=False)
        text = getDocumentProperties(filenameResources, variableparams, debug=False)

        text.calcVariables( resources )

        for i in range(len(variableparams)+1):
            #variable, params = variableparams[i]
            varlabel = variables[i]
            # squeeze the filename in first:
            value = filename
            if i > 0:
                value = text.variablevalues[i-1]
            print varlabel, "\t", str(value)
        print 'level', "\t", text.level

        allobservedtrees.update(text.trees.keys())
        treespertext.append( (set(text.trees.keys()), text.level) )
        #print "mwpw", text.getMeanWeightPerWord()
        outputRows.append( [baseFileName] + text.variablevalues + [text.level] )


    savetoArff(outarffdir, corpusName, headerInfo, variables, outputRows )
    savetoArff(outarffdir, corpusName + "class", headerInfo, variables, outputRows, levelAsClass=True )

    arfftreefile = "testtrees"
    #corpusName = "test"
    #headerInfo = "a test corpus\n of stuff"
    treevariables = list(allobservedtrees)
    #print allobservedtrees
    #print treespertext
    treeoutputRows = [[1 if x in trees else 0 for x in treevariables] + [level] for trees, level in treespertext]
    treevariables.append("level")
    #print "ntrees", len(treevariables)
    #for a,b in zip(treevariables, treeoutputRows):
    #    print a,len(b), b

    savetoTreeArff(outarffdir, arfftreefile, headerInfo, treevariables, treeoutputRows, levelAsClass=True )