def test():
bigrams = {} # bigram as key, frequency as value
tokens = {} # token as key, frequency as value
tokencount = 0 # number of tokens
bigramcount = 0 # number of bigrams
alphabet = "" # all characters used
for i in sys.argv[1:]:
for x in glob.glob(os.path.normcase(i)):
try:
file = open(x, "r")
for i in file.readlines():
#i = string.lower(string.strip(i))
i = i.strip().lower()
if i == "":
continue
wordlist = getWordList(i)
bigrams, bigramcount = getBigrams(wordlist, bigrams, bigramcount)
tokens, tokencount = getTokens(wordlist, tokens, tokencount)
file.close()
except IOError:
file.close()
print("Got total:\nBigrams: " + str(bigramcount) + "\nTokens: " + str(tokencount))
print("Bigram\tFrequency\tRelative Frequency\tMutual Information\tRelative Entropy")
#myTokens = string.split(i[0])
for i in sortNgrams(bigrams):
tokenlist = list(i)[0].split()
re = RE(rF(i[1], bigramcount), P(tokenlist[1], tokens, tokencount), P(tokenlist[0], tokens, tokencount))
#print(i[0] + "\t" + str(i[1]) + "\t" + str(rF(i[1], bigramcount)) + "\t" + str(MI(i[0], rF(i[1], bigramcount), tokens, tokencount)) + "\t" + str(RE(i[0], rF(i[1], bigramcount), rF(myTokens[1], tokencount), rF(myTokens[0], tokencount))))
print(i[0] + "\t" + str(i[1]) + "\t" + str(rF(i[1], bigramcount)) + "\t" + str(MI(i[0], rF(i[1], bigramcount), tokens, tokencount)) + "\t" + str(re))
def caculate(filename, freq=100):
bigrams = {} # bigram as key, frequency as value
tokens = {} # token as key, frequency as value
tokencount = 0 # number of tokens
bigramcount = 0 # number of bigrams
alphabet = "" # all characters used
try:
file = open(filename, "r")
for i in file.readlines():
i = i.strip().lower()
if i == "":
continue
wordlist = getWordList(i)
bigrams, bigramcount = getBigrams(wordlist, bigrams, bigramcount)
tokens, tokencount = getTokens(wordlist, tokens, tokencount)
file.close()
except IOError:
file.close()
if os.path.exists("mi.txt"):
os.remove('mi.txt')
if os.path.exists("dict.txt"):
os.remove('dict.txt')
f = open("mi.txt", "w")
fl = open("dict.txt", "w")
print("Got total:\nBigrams: " + str(bigramcount) + "\nTokens: " + str(tokencount))
#print("Bigram\tFrequency\tRelative Frequency\tMutual Information\tRelative Entropy")
f.write("T1, T2, Frequency, Relative Frequency, Mutual Information, Relative Entropy\n")
sep = ", "
for i in sortNgrams(bigrams):
tokenlist = list(i)[0].split()
re = RE(rF(i[1], bigramcount), P(tokenlist[1], tokens, tokencount), P(tokenlist[0], tokens, tokencount))
mi = MI(i[0], rF(i[1], bigramcount), tokens, tokencount)
if mi > freq:
f.write(tokenlist[1] + sep + tokenlist[0] + sep + str(i[1]) + sep + str(rF(i[1], bigramcount)) + sep + str(mi) + sep + str(re) + "\n")
fl.write(tokenlist[1] + " " + tokenlist[0]+ "\n")
f.close()
fl.close()
ret = merge("dict.txt", "data.basket")
'''
def PRE(token): """Relative entropy is P1*log2(P1/P2) where here P1 is P(rightword) and P2 is P(right word | left word).""" global bigrams, tokens, bigramsleft, bigramsright, tokencount, bigramcount rre = 0.0 lre = 0.0 tokensleft = [] tokensright = [] # all tokens left of token if bigramsleft.has_key(token): tokensleft = bigramsleft[token] # all tokens right of token if bigramsright.has_key(token): tokensright = bigramsright[token] count = 0 for x in tokensleft: count += 1 lre += RE(rF(bigrams[x+" "+token], bigramcount), rF(tokens[token], tokencount), rF(types[x], typecount)) count = 0 for x in tokensright: count += 1 rre += RE(rF(bigrams[token+" "+x], bigramcount), rF(tokens[token], tokencount), rF(types[x], typecount)) return rre, lre
return bigramprob * math.log(bigramprob/(px * py) , 2)
if __name__ == "__main__":
bigrams = {} # bigram as key, frequency as value
tokens = {} # token as key, frequency as value
tokencount = 0 # number of tokens
bigramcount = 0 # number of bigrams
alphabet = "" # all characters used
for i in sys.argv[1:]:
for x in glob.glob(os.path.normcase(i)):
try:
file = open(x, "r")
for i in file.readlines():
i = string.lower(string.strip(i))
if i == "":
continue
wordlist = getWordList(i)
bigrams, bigramcount = getBigrams(wordlist, bigrams, bigramcount)
tokens, tokencount = getTokens(wordlist, tokens, tokencount)
file.close()
except IOError:
file.close()
print "Got total:\nBigrams: " + str(bigramcount) + "\nTokens: " + str(tokencount)
print "Bigram\tFrequency\tRelative Frequency\tMutual Information"
myTokens = string.split(i[0])
for i in sortNgrams(bigrams):
print i[0] + "\t" + str(i[1]) + "\t" + str(rF(i[1], bigramcount)) + "\t" + str(MI(i[0], rF(i[1], bigramcount), tokens, tokencount)) + "\t" + str(RE(i[0], rF(i[1], bigramcount), rF(myTokens[1], tokencount), rF(myTokens[0], tokencount)))
rre += RE(rF(bigrams[token+" "+x], bigramcount), rF(tokens[token], tokencount), rF(types[x], typecount)) return rre, lre if __name__ == "__main__": for i in sys.argv[1:]: for x in glob.glob(os.path.normcase(i)): try: file = open(x, "r") for i in file.readlines(): i = string.lower(string.strip(i)) if i == "": continue wordlist = getTWordList(i) bigrams, bigramcount = getTBigrams(wordlist, bigrams, bigramcount, TOKEN, TYPE) tokens, tokencount = getTTokens(wordlist, tokens, tokencount, TOKEN) types, typecount = getTTokens(wordlist, types, typecount, TYPE) bigramsleft, bigramsright = getTLRBigrams(wordlist, bigramsleft, bigramsright, TOKEN, TYPE) file.close() except IOError: file.close() myTokens = sortNgrams(tokens) print "Left RE\tToken\tRight RE\tFrequency\tRelative Frequency" for x in range(min(len(myTokens), PRINTWORDS)): rre, lre = PRE(myTokens[x][0]) print str(lre) + "\t" + myTokens[x][0] + "\t" + str(rre) + "\t" + str(myTokens[x][1]) + "\t" + str(rF(myTokens[x][1], tokencount))
