def __init__(self, filepath):
"""Initialize FrequencyStrategy"""
self.possible = WordSet()
# Cache of possible words given current game state.
# Initially just all the possible words, divided out based on length.
# As games progress, more WordSets-possible-given-game-state-X are added.
self.wordCache = collections.defaultdict(WordSet)
# process dictionary file
self.parseWordsFile(filepath)
self.seedCache()
self.newGame()
def __init__(self, streets_file, towns_file, country_file):
super(AddressDetector, self).__init__('Detect addresses')
self.streets = WLTStorage(streets_file)
self.towns = WLTStorage(towns_file)
self.countries = WordSet(country_file)
# word: The string in which the word being modified is stored.
# punctuation: Tuple containing punctuation that is to be removed.
def removePunctuation(word, punctuation):
for x in punctuation:
if (word[len(word) - 1] == x):
word = word[0:len(word) - 1]
return word
#Main
#st = LancasterStemmer()
con = sql.connect("UselessWords.db")
cur = con.cursor()
#Word_List stores words from the input parapgraph.
Word_List = WordSet()
paragraph = input("Enter the text to be summarized: ")
#tokens: list of words in the input parargraph.
tokens = paragraph.split()
#Removes formatting from each word then adds it to Word_List.
for item in tokens:
item = item.strip()
item = removePunctuation(item, punctuation)
#x = st.stem(x)
if (not useless(cur, item)):
Word_List.add(Word(item, None, 1))
#output stores the Word Object with the highest count
output = Word_List.max()
print(output.word)
con.close()
def __init__(self, ignore_list):
super(AbbrDetector, self).__init__('Detect abbreviations')
self.ignored_words = WordSet(ignore_list, filt=lambda x: x.lower())
self.cache = set()
def generateOutputFile(developmentSetFilename, testSetFilename, firstInputWord,
secondInputWord, outputFilename):
print "Started with: "
print "\tDevelopment set filename: %s" % developmentSetFilename
print "\tTest set filename: %s" % testSetFilename
print "\tInput word: %s" % firstInputWord
print "\tInput word2: %s" % secondInputWord
print "\tOutput filename: %s" % outputFilename
vocabularySize = 300000
file = open(outputFilename, "w+")
file.write("#Students:\tSaar Arbel\tBoaz Berman\t315681775\t311504401\n")
file.write("Output1: " + developmentSetFilename + "\n")
file.write("Output2: " + testSetFilename + "\n")
file.write("Output3: " + firstInputWord + " " + secondInputWord + "\n")
file.write("Output4: " + outputFilename + "\n")
file.write("Output5: " + str(vocabularySize) + "\n")
with open(developmentSetFilename, 'rb') as input_file:
input_file_data = input_file.read()
words = parse_file_data(input_file_data)
cuttingIndex = int(round(len(words) * 0.9))
trainingSet, validationSet = words[:cuttingIndex], words[cuttingIndex:]
trainingWordSet, validationWordSet = WordSet(
trainingSet, vocabularySize), WordSet(validationSet, vocabularySize)
file.write("Output6: " + str(len(words)) + "\n")
file.write("Output7: " + str(validationWordSet.length) + "\n")
file.write("Output8: " + str(trainingWordSet.length) + "\n")
file.write("Output9: " + str(trainingWordSet.distinctLength) + "\n")
file.write("Output10: " +
str(trainingWordSet.countAppearances(firstInputWord)) + "\n")
trainingBigramWordSet = BigramWordSet(trainingSet, vocabularySize,
trainingWordSet)
file.write("Output11: " + str(
trainingBigramWordSet.countAppearances(firstInputWord,
secondInputWord)) + "\n")
validationBigramWordSet = BigramWordSet(validationSet, vocabularySize,
validationWordSet)
backOffTrainingModel = BackOffModel(trainingBigramWordSet, trainingWordSet)
backOffValidationModel = BackOffModel(validationBigramWordSet,
validationWordSet)
print str(
backOffTrainingModel.bigramWordSet.pLidstone(
("bank", "economist"), 0.001)) + " boaz"
print backOffTrainingModel.pBackOff("bank", "economist", 0.1)
print "Debug %f" % backOffTrainingModel.debug()
file.write('Output12: ' + str(
backOffPerplexity(backOffTrainingModel, backOffValidationModel,
0.0001)) + "\n")
print "finished 12"
file.write('Output13: ' + str(
backOffPerplexity(backOffTrainingModel, backOffValidationModel, 0.001))
+ "\n")
print "finished 13"
file.write('Output14: ' + str(
backOffPerplexity(backOffTrainingModel, backOffValidationModel, 0.1)) +
"\n")
print "finished 14"
minperplexity, minlamda = minimumPerplexity(backOffTrainingModel,
backOffValidationModel)
file.write('Output15: ' + str(minlamda) + "\n")
print "finished 15"
file.write('Output16: ' + str(minperplexity) + "\n")
print "finished 16"
with open(testSetFilename, 'rb') as input_file2:
input_file_data2 = input_file2.read()
words2 = parse_file_data(input_file_data2)
trainingWordSet2 = WordSet(words2, vocabularySize)
trainingBigramWordSet2 = BigramWordSet(words2, vocabularySize,
trainingWordSet2)
backOffTrainingModel2 = BackOffModel(trainingBigramWordSet2,
trainingWordSet2)
file.write('Output17: ' + str(
backOffPerplexity(backOffTrainingModel, backOffTrainingModel2, 0.0003))
+ "\n")
print "finished 17"
file.write('Output18: ' +
str(printTable(backOffTrainingModel, 0.001, firstInputWord)))