def get_word_list(chain, pre, randomizer, word_count, NONWORD):
t = suffix.choose_word(chain, pre, randomizer)
if word_count == 0 or t == NONWORD:
return tuple()
else:
return (suffix.choose_word(chain, pre, randomizer), ) + get_word_list(
chain, prefix.shift_in(pre, t), randomizer, word_count - 1,
NONWORD)
def words_gen(chain, current_prefix, randomizer, counter, nonword): # Recursively generate a random list of words
new_word = suffix.choose_word(chain, current_prefix, randomizer)
if new_word == nonword:
return [] # If the next word is end of line, stop early
elif counter == 1:
return [new_word] # Generate last word in sentence
else:
# Recursively generate new words until word count reached
return [new_word] + words_gen(chain, prefix.shift_in(current_prefix, new_word), randomizer, counter-1, nonword)
def pairs_gen(input_file, linegen): # Generate pairs of prefixes and suffixes from lines of text
current_pair = prefix.new_prefix(NONWORD, NONWORD) # Start with an empty prefix
for line in linegen(input_file):
words_list = line.split(" ") # Split line of text into words separated by spaces
while len(words_list) > 0:
new_word = words_list.pop(0) # As long as there are words left, pop one from the list
yield (current_pair, new_word) # Yield the current state and the next word
current_pair = prefix.shift_in(current_pair, new_word) # Shift the new word into the current pair
yield (current_pair, NONWORD) # The final pair ends with a NONWORD
def pairs_gen(fileName, lineGenerator):
generator = lineGenerator(fileName)
currPrefix = prefix.new_prefix(NONWORD, NONWORD)
for line in generator:
words = line.split()
for word in words:
yield (currPrefix, word)
currPrefix = prefix.shift_in(currPrefix, word)
yield (currPrefix, NONWORD)
def get_word_list_helper(int, lis, chain, pre, randomizer, NONWORD):
if int is 0:
return lis
word = suffix.choose_word(chain, pre, randomizer)
if word is NONWORD:
return lis
lis.append(word)
return get_word_list_helper(int - 1, lis, chain,
prefix.shift_in(pre,
word), randomizer, NONWORD)
def pairs_gen(file, generator):
pre = (NONWORD, NONWORD)
gen = generator(file)
for line in gen:
words = line.split(" ")
for word in words:
tup = (pre, word)
pre = prefix.shift_in(pre, word)
yield tup
words = next(generator(file)).split(" ")
def add_to_word_list(trio_list, x):
prev_trio = trio_list[-1]
prev_prefix = prev_trio[0]
prev_n = prev_trio[2]
if prev_n <= 0:
return trio_list
else:
new_word = suffix.choose_word(chain, prev_prefix, randomizer_fn)
new_prefix = prefix.shift_in(prev_prefix, new_word)
if new_word == NONWORD:
return trio_list + [(new_prefix, "", 0)]
else:
return trio_list + [(new_prefix, new_word, prev_n - 1)]