def setVocabularyMeasures(self):
     lemmacats = list(itertools.chain(*[x.listLemmaCats() for x in self.paragraphs]))
     try:
         # as the fit can crash
         popt, pcov = calcPLex.calcPLex( lemmacats, lemmacat2freqrank)
         self.PLex = popt[0]
     except:
         self.PLex = 0.0
     #print lemmacats
     popt, pcov = calcPLex.calcS( lemmacats, lemmacat2freqrank)
     self.S = popt[0]
     if len(lemmacats) >= 50:
         self.vocd = calcPLex.getVOCD(lemmacats)
     else:
         self.vocd = 0.0
     self.mtld = calcPLex.getMTLD(lemmacats)
     if len(lemmacats) >= 42:
         self.hdd = calcHDD.calcHDD(lemmacats)
     else:
         self.hdd = 0.0
     #self.maas = calcHDD.calcMaas(lemmacats) # don't think this works right atm
     vocabs, self.vocabOther, self.vocabUnk =\
         calcPLex.calcLFP( lemmacats, lemmacat2freqrank, difficultybins = (1000,2000,3000,4000,8000))
     self.vocab1k, self.vocab2k, self.vocab3k, self.vocab4k, self.vocab8k = vocabs
for testxml in testxmls:
    print "loading xml", testxml
    tok2finalforms, tok2lemmacats, verb2info, trees, (weight, maxweight) = getFinalTokenFormsAndTreesAndWeight(testxml)
    # each tok2lemmacat goes to a list to 2-tuples
    # print tok2lemmacats.values()
    for lemmacats in tok2lemmacats.values():
        wordforms.extend([x[0] + u"_" + x[1] for x in lemmacats])

print wordforms
print "tokens", len(wordforms)
print "types", len(set(wordforms))
popt, pcov = calcPLex.calcPLex( wordforms, lemmacat2freqrank)

print "popt", popt
print "pcov", pcov

print "S"
popt, pcov = calcPLex.calcS( wordforms, lemmacat2freqrank)

print "popt", popt
print "pcov", pcov


vocd = calcPLex.getVOCD( wordforms )
mtld = calcPLex.getMTLD( wordforms )

print "vocd:", vocd
print "mtld:", mtld