def test(): bigrams = {} # bigram as key, frequency as value tokens = {} # token as key, frequency as value tokencount = 0 # number of tokens bigramcount = 0 # number of bigrams alphabet = "" # all characters used for i in sys.argv[1:]: for x in glob.glob(os.path.normcase(i)): try: file = open(x, "r") for i in file.readlines(): #i = string.lower(string.strip(i)) i = i.strip().lower() if i == "": continue wordlist = getWordList(i) bigrams, bigramcount = getBigrams(wordlist, bigrams, bigramcount) tokens, tokencount = getTokens(wordlist, tokens, tokencount) file.close() except IOError: file.close() print("Got total:\nBigrams: " + str(bigramcount) + "\nTokens: " + str(tokencount)) print("Bigram\tFrequency\tRelative Frequency\tMutual Information\tRelative Entropy") #myTokens = string.split(i[0]) for i in sortNgrams(bigrams): tokenlist = list(i)[0].split() re = RE(rF(i[1], bigramcount), P(tokenlist[1], tokens, tokencount), P(tokenlist[0], tokens, tokencount)) #print(i[0] + "\t" + str(i[1]) + "\t" + str(rF(i[1], bigramcount)) + "\t" + str(MI(i[0], rF(i[1], bigramcount), tokens, tokencount)) + "\t" + str(RE(i[0], rF(i[1], bigramcount), rF(myTokens[1], tokencount), rF(myTokens[0], tokencount)))) print(i[0] + "\t" + str(i[1]) + "\t" + str(rF(i[1], bigramcount)) + "\t" + str(MI(i[0], rF(i[1], bigramcount), tokens, tokencount)) + "\t" + str(re))
def caculate(filename, freq=100): bigrams = {} # bigram as key, frequency as value tokens = {} # token as key, frequency as value tokencount = 0 # number of tokens bigramcount = 0 # number of bigrams alphabet = "" # all characters used try: file = open(filename, "r") for i in file.readlines(): i = i.strip().lower() if i == "": continue wordlist = getWordList(i) bigrams, bigramcount = getBigrams(wordlist, bigrams, bigramcount) tokens, tokencount = getTokens(wordlist, tokens, tokencount) file.close() except IOError: file.close() if os.path.exists("mi.txt"): os.remove('mi.txt') if os.path.exists("dict.txt"): os.remove('dict.txt') f = open("mi.txt", "w") fl = open("dict.txt", "w") print("Got total:\nBigrams: " + str(bigramcount) + "\nTokens: " + str(tokencount)) #print("Bigram\tFrequency\tRelative Frequency\tMutual Information\tRelative Entropy") f.write("T1, T2, Frequency, Relative Frequency, Mutual Information, Relative Entropy\n") sep = ", " for i in sortNgrams(bigrams): tokenlist = list(i)[0].split() re = RE(rF(i[1], bigramcount), P(tokenlist[1], tokens, tokencount), P(tokenlist[0], tokens, tokencount)) mi = MI(i[0], rF(i[1], bigramcount), tokens, tokencount) if mi > freq: f.write(tokenlist[1] + sep + tokenlist[0] + sep + str(i[1]) + sep + str(rF(i[1], bigramcount)) + sep + str(mi) + sep + str(re) + "\n") fl.write(tokenlist[1] + " " + tokenlist[0]+ "\n") f.close() fl.close() ret = merge("dict.txt", "data.basket") '''
def PRE(token): """Relative entropy is P1*log2(P1/P2) where here P1 is P(rightword) and P2 is P(right word | left word).""" global bigrams, tokens, bigramsleft, bigramsright, tokencount, bigramcount rre = 0.0 lre = 0.0 tokensleft = [] tokensright = [] # all tokens left of token if bigramsleft.has_key(token): tokensleft = bigramsleft[token] # all tokens right of token if bigramsright.has_key(token): tokensright = bigramsright[token] count = 0 for x in tokensleft: count += 1 lre += RE(rF(bigrams[x+" "+token], bigramcount), rF(tokens[token], tokencount), rF(types[x], typecount)) count = 0 for x in tokensright: count += 1 rre += RE(rF(bigrams[token+" "+x], bigramcount), rF(tokens[token], tokencount), rF(types[x], typecount)) return rre, lre
return bigramprob * math.log(bigramprob/(px * py) , 2) if __name__ == "__main__": bigrams = {} # bigram as key, frequency as value tokens = {} # token as key, frequency as value tokencount = 0 # number of tokens bigramcount = 0 # number of bigrams alphabet = "" # all characters used for i in sys.argv[1:]: for x in glob.glob(os.path.normcase(i)): try: file = open(x, "r") for i in file.readlines(): i = string.lower(string.strip(i)) if i == "": continue wordlist = getWordList(i) bigrams, bigramcount = getBigrams(wordlist, bigrams, bigramcount) tokens, tokencount = getTokens(wordlist, tokens, tokencount) file.close() except IOError: file.close() print "Got total:\nBigrams: " + str(bigramcount) + "\nTokens: " + str(tokencount) print "Bigram\tFrequency\tRelative Frequency\tMutual Information" myTokens = string.split(i[0]) for i in sortNgrams(bigrams): print i[0] + "\t" + str(i[1]) + "\t" + str(rF(i[1], bigramcount)) + "\t" + str(MI(i[0], rF(i[1], bigramcount), tokens, tokencount)) + "\t" + str(RE(i[0], rF(i[1], bigramcount), rF(myTokens[1], tokencount), rF(myTokens[0], tokencount)))
rre += RE(rF(bigrams[token+" "+x], bigramcount), rF(tokens[token], tokencount), rF(types[x], typecount)) return rre, lre if __name__ == "__main__": for i in sys.argv[1:]: for x in glob.glob(os.path.normcase(i)): try: file = open(x, "r") for i in file.readlines(): i = string.lower(string.strip(i)) if i == "": continue wordlist = getTWordList(i) bigrams, bigramcount = getTBigrams(wordlist, bigrams, bigramcount, TOKEN, TYPE) tokens, tokencount = getTTokens(wordlist, tokens, tokencount, TOKEN) types, typecount = getTTokens(wordlist, types, typecount, TYPE) bigramsleft, bigramsright = getTLRBigrams(wordlist, bigramsleft, bigramsright, TOKEN, TYPE) file.close() except IOError: file.close() myTokens = sortNgrams(tokens) print "Left RE\tToken\tRight RE\tFrequency\tRelative Frequency" for x in range(min(len(myTokens), PRINTWORDS)): rre, lre = PRE(myTokens[x][0]) print str(lre) + "\t" + myTokens[x][0] + "\t" + str(rre) + "\t" + str(myTokens[x][1]) + "\t" + str(rF(myTokens[x][1], tokencount))