def __init__(self, brain_db, chain_length):
self.brain_db = brain_db
self.chain_length = chain_length
# Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
self.sqlite_dict = DatabaseDictionary(self.brain_db,
MARKOV_VALUE_PROPS,
MarkovDictionaryValue)
def load_brain(self):
print 'Brain loading...'
as_process(markov_dictionary_from_file,
(self.temp_db_file, self.brain_file, self.chain_length
)) # Shields main process from intermediate memory used
self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
print 'Brain loaded.'
print 'Markov dictionary has %i keys' % (len(self.markov), )
def sync_with_file(self, brain_file):
if len(self.sqlite_dict) > 0:
return
self.sqlite_dict.close()
as_process(markov_dictionary_from_file,
(self.brain_db, brain_file, self.chain_length))
self.sqlite_dict = DatabaseDictionary(self.brain_db,
MARKOV_VALUE_PROPS,
MarkovDictionaryValue)
class MarkovCoreSqlite():
def __init__(self, brain_db, chain_length):
self.brain_db = brain_db
self.chain_length = chain_length
# Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
self.sqlite_dict = DatabaseDictionary(self.brain_db,
MARKOV_VALUE_PROPS,
MarkovDictionaryValue)
def sync_with_file(self, brain_file):
if len(self.sqlite_dict) > 0:
return
self.sqlite_dict.close()
as_process(markov_dictionary_from_file,
(self.brain_db, brain_file, self.chain_length))
self.sqlite_dict = DatabaseDictionary(self.brain_db,
MARKOV_VALUE_PROPS,
MarkovDictionaryValue)
def add_to_markov_dictionary(self, line):
self.sqlite_dict.begin()
for c in xrange(1, self.chain_length + 1):
add_to_markov_dictionary(self.sqlite_dict, c, line)
self.sqlite_dict.commit()
def generate_sentence(self, seed_msg, max_words):
return generate_sentence(self.sqlite_dict, seed_msg, self.chain_length,
max_words)
def close(self):
self.sqlite_dict.close()
def markov_dictionary_from_file(temp_db_file, brain_file, chain_length):
print 'Creating markov chains from %s' % brain_file
line_count = count_lines(brain_file)
temp_dict = {}
for c in xrange(1, chain_length + 1):
print 'Creating markov chains of length %i' % c
with open(brain_file, 'r') as f:
progress_bar = ProgressBar(line_count)
for line in f:
add_to_markov_dictionary(temp_dict, c,
line.strip().decode('utf-8').lower())
progress_bar.update()
print 'Populating database dictionary with markov chains'
db_dict = DatabaseDictionary(temp_db_file, MARKOV_VALUE_PROPS)
db_dict.begin()
db_dict.replace(temp_dict)
db_dict.commit()
db_dict.close()
class MarkovBrain():
new_brain_lines_limit = 1024
def __init__(self, brain_file, chain_length, max_words):
self.brain_file = brain_file
self.chain_length = chain_length
self.max_words = max_words
temp_f, self.temp_db_file = tempfile.mkstemp()
os.close(temp_f)
print 'Using %s as temporary database dictionary file' % self.temp_db_file
self.markov = None # Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
self.new_brain_lines = [
] # New lines seen since brain was loaded. Will be added to brain file when size reaches new_brain_lines_limit
self.load_brain()
def load_brain(self):
print 'Brain loading...'
as_process(markov_dictionary_from_file,
(self.temp_db_file, self.brain_file, self.chain_length
)) # Shields main process from intermediate memory used
self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
print 'Brain loaded.'
print 'Markov dictionary has %i keys' % (len(self.markov), )
def __add_new_brain_line(self, msg):
self.new_brain_lines.append(msg + '\n')
if len(self.new_brain_lines) >= self.new_brain_lines_limit:
self.__dump_new_brain_lines()
def __dump_new_brain_lines(self):
with open(self.brain_file, 'a') as f:
for line in self.new_brain_lines:
f.write(line.encode('utf-8'))
print '%i new brain lines dumped.' % (len(self.new_brain_lines))
self.new_brain_lines = []
@time_function
def add_to_brain(self, original_msg):
msg = original_msg.replace('\x00', '').strip().decode('utf-8')
# Don't bother with empty lines.
if len(msg) == 0:
return
self.__add_new_brain_line(msg)
self.markov.begin()
add_to_markov_dictionary(self.markov, self.chain_length, msg)
self.markov.commit()
def generate_sentence(self, seed_msg):
return generate_sentence(self.markov, seed_msg, self.chain_length,
self.max_words)
def close(self):
self.__dump_new_brain_lines()
self.markov.close()
os.remove(self.temp_db_file)
def markov_dictionary_from_file(temp_db_file, brain_file, chain_length):
print 'Creating markov chains from %s' % brain_file
line_count = count_lines(brain_file)
temp_dict = {}
for c in xrange(1, chain_length + 1):
print 'Creating markov chains of length %i' % c
with open(brain_file, 'r') as f:
progress_bar = ProgressBar(line_count)
for line in f:
add_to_markov_dictionary(temp_dict, c, line.strip().decode('utf-8').lower())
progress_bar.update()
print 'Populating database dictionary with markov chains'
db_dict = DatabaseDictionary(temp_db_file, MARKOV_VALUE_PROPS)
db_dict.begin()
db_dict.replace(temp_dict)
db_dict.commit()
db_dict.close()
class MarkovBrain():
new_brain_lines_limit = 1024
def __init__(self, brain_file, chain_length, max_words):
self.brain_file = brain_file
self.chain_length = chain_length
self.max_words = max_words
temp_f, self.temp_db_file = tempfile.mkstemp()
os.close(temp_f)
print 'Using %s as temporary database dictionary file' % self.temp_db_file
self.markov = None # Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
self.new_brain_lines = [] # New lines seen since brain was loaded. Will be added to brain file when size reaches new_brain_lines_limit
self.load_brain()
def load_brain(self):
print 'Brain loading...'
as_process(markov_dictionary_from_file, (self.temp_db_file, self.brain_file, self.chain_length)) # Shields main process from intermediate memory used
self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
print 'Brain loaded.'
print 'Markov dictionary has %i keys' % (len(self.markov),)
def __add_new_brain_line(self, msg):
self.new_brain_lines.append(msg + '\n')
if len(self.new_brain_lines) >= self.new_brain_lines_limit:
self.__dump_new_brain_lines()
def __dump_new_brain_lines(self):
with open(self.brain_file, 'a') as f:
for line in self.new_brain_lines:
f.write(line.encode('utf-8'))
print '%i new brain lines dumped.' % (len(self.new_brain_lines))
self.new_brain_lines = []
@time_function
def add_to_brain(self, original_msg):
msg = original_msg.replace('\x00', '').strip().decode('utf-8')
# Don't bother with empty lines.
if len(msg) == 0:
return
self.__add_new_brain_line(msg)
self.markov.begin()
add_to_markov_dictionary(self.markov, self.chain_length, msg)
self.markov.commit()
def generate_sentence(self, seed_msg):
return generate_sentence(self.markov, seed_msg, self.chain_length, self.max_words)
def close(self):
self.__dump_new_brain_lines()
self.markov.close()
os.remove(self.temp_db_file)
def load_brain(self):
print 'Brain loading...'
as_process(markov_dictionary_from_file, (self.temp_db_file, self.brain_file, self.chain_length)) # Shields main process from intermediate memory used
self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
print 'Brain loaded.'
print 'Markov dictionary has %i keys' % (len(self.markov),)