Example #1
0
    def __init__(self, brain_db, chain_length):
        self.brain_db = brain_db
        self.chain_length = chain_length

        # Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
        self.sqlite_dict = DatabaseDictionary(self.brain_db,
                                              MARKOV_VALUE_PROPS,
                                              MarkovDictionaryValue)
Example #2
0
 def load_brain(self):
     print 'Brain loading...'
     as_process(markov_dictionary_from_file,
                (self.temp_db_file, self.brain_file, self.chain_length
                 ))  # Shields main process from intermediate memory used
     self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
     print 'Brain loaded.'
     print 'Markov dictionary has %i keys' % (len(self.markov), )
Example #3
0
 def sync_with_file(self, brain_file):
     if len(self.sqlite_dict) > 0:
         return
     self.sqlite_dict.close()
     as_process(markov_dictionary_from_file,
                (self.brain_db, brain_file, self.chain_length))
     self.sqlite_dict = DatabaseDictionary(self.brain_db,
                                           MARKOV_VALUE_PROPS,
                                           MarkovDictionaryValue)
Example #4
0
class MarkovCoreSqlite():
    def __init__(self, brain_db, chain_length):
        self.brain_db = brain_db
        self.chain_length = chain_length

        # Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
        self.sqlite_dict = DatabaseDictionary(self.brain_db,
                                              MARKOV_VALUE_PROPS,
                                              MarkovDictionaryValue)

    def sync_with_file(self, brain_file):
        if len(self.sqlite_dict) > 0:
            return
        self.sqlite_dict.close()
        as_process(markov_dictionary_from_file,
                   (self.brain_db, brain_file, self.chain_length))
        self.sqlite_dict = DatabaseDictionary(self.brain_db,
                                              MARKOV_VALUE_PROPS,
                                              MarkovDictionaryValue)

    def add_to_markov_dictionary(self, line):
        self.sqlite_dict.begin()
        for c in xrange(1, self.chain_length + 1):
            add_to_markov_dictionary(self.sqlite_dict, c, line)
        self.sqlite_dict.commit()

    def generate_sentence(self, seed_msg, max_words):
        return generate_sentence(self.sqlite_dict, seed_msg, self.chain_length,
                                 max_words)

    def close(self):
        self.sqlite_dict.close()
Example #5
0
def markov_dictionary_from_file(temp_db_file, brain_file, chain_length):
    print 'Creating markov chains from %s' % brain_file

    line_count = count_lines(brain_file)

    temp_dict = {}
    for c in xrange(1, chain_length + 1):
        print 'Creating markov chains of length %i' % c
        with open(brain_file, 'r') as f:
            progress_bar = ProgressBar(line_count)
            for line in f:
                add_to_markov_dictionary(temp_dict, c,
                                         line.strip().decode('utf-8').lower())
                progress_bar.update()

    print 'Populating database dictionary with markov chains'
    db_dict = DatabaseDictionary(temp_db_file, MARKOV_VALUE_PROPS)
    db_dict.begin()
    db_dict.replace(temp_dict)
    db_dict.commit()
    db_dict.close()
Example #6
0
class MarkovBrain():
    new_brain_lines_limit = 1024

    def __init__(self, brain_file, chain_length, max_words):
        self.brain_file = brain_file
        self.chain_length = chain_length
        self.max_words = max_words

        temp_f, self.temp_db_file = tempfile.mkstemp()
        os.close(temp_f)
        print 'Using %s as temporary database dictionary file' % self.temp_db_file
        self.markov = None  # Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
        self.new_brain_lines = [
        ]  # New lines seen since brain was loaded. Will be added to brain file when size reaches new_brain_lines_limit

        self.load_brain()

    def load_brain(self):
        print 'Brain loading...'
        as_process(markov_dictionary_from_file,
                   (self.temp_db_file, self.brain_file, self.chain_length
                    ))  # Shields main process from intermediate memory used
        self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
        print 'Brain loaded.'
        print 'Markov dictionary has %i keys' % (len(self.markov), )

    def __add_new_brain_line(self, msg):
        self.new_brain_lines.append(msg + '\n')
        if len(self.new_brain_lines) >= self.new_brain_lines_limit:
            self.__dump_new_brain_lines()

    def __dump_new_brain_lines(self):
        with open(self.brain_file, 'a') as f:
            for line in self.new_brain_lines:
                f.write(line.encode('utf-8'))
        print '%i new brain lines dumped.' % (len(self.new_brain_lines))
        self.new_brain_lines = []

    @time_function
    def add_to_brain(self, original_msg):
        msg = original_msg.replace('\x00', '').strip().decode('utf-8')

        # Don't bother with empty lines.
        if len(msg) == 0:
            return

        self.__add_new_brain_line(msg)
        self.markov.begin()
        add_to_markov_dictionary(self.markov, self.chain_length, msg)
        self.markov.commit()

    def generate_sentence(self, seed_msg):
        return generate_sentence(self.markov, seed_msg, self.chain_length,
                                 self.max_words)

    def close(self):
        self.__dump_new_brain_lines()
        self.markov.close()
        os.remove(self.temp_db_file)
Example #7
0
def markov_dictionary_from_file(temp_db_file, brain_file, chain_length):
    print 'Creating markov chains from %s' % brain_file

    line_count = count_lines(brain_file)

    temp_dict = {}
    for c in xrange(1, chain_length + 1):
        print 'Creating markov chains of length %i' % c
        with open(brain_file, 'r') as f:
            progress_bar = ProgressBar(line_count)
            for line in f:
                add_to_markov_dictionary(temp_dict, c, line.strip().decode('utf-8').lower())
                progress_bar.update()

    print 'Populating database dictionary with markov chains'
    db_dict = DatabaseDictionary(temp_db_file, MARKOV_VALUE_PROPS)
    db_dict.begin()
    db_dict.replace(temp_dict)
    db_dict.commit()
    db_dict.close()
Example #8
0
class MarkovBrain():
    new_brain_lines_limit = 1024

    def __init__(self, brain_file, chain_length, max_words):
        self.brain_file = brain_file
        self.chain_length = chain_length
        self.max_words = max_words

        temp_f, self.temp_db_file = tempfile.mkstemp()
        os.close(temp_f)
        print 'Using %s as temporary database dictionary file' % self.temp_db_file
        self.markov = None # Holds markov chain data. key(tuple(words[chain_length])) -> {word_choice1: count, word_choice2: count}
        self.new_brain_lines = [] # New lines seen since brain was loaded. Will be added to brain file when size reaches new_brain_lines_limit

        self.load_brain()

    def load_brain(self):
        print 'Brain loading...'
        as_process(markov_dictionary_from_file, (self.temp_db_file, self.brain_file, self.chain_length)) # Shields main process from intermediate memory used
        self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
        print 'Brain loaded.'
        print 'Markov dictionary has %i keys' % (len(self.markov),)

    def __add_new_brain_line(self, msg):
        self.new_brain_lines.append(msg + '\n')
        if len(self.new_brain_lines) >= self.new_brain_lines_limit:
            self.__dump_new_brain_lines()

    def __dump_new_brain_lines(self):
        with open(self.brain_file, 'a') as f:
            for line in self.new_brain_lines:
                f.write(line.encode('utf-8'))
        print '%i new brain lines dumped.' % (len(self.new_brain_lines))
        self.new_brain_lines = []

    @time_function
    def add_to_brain(self, original_msg):
        msg = original_msg.replace('\x00', '').strip().decode('utf-8')

        # Don't bother with empty lines.
        if len(msg) == 0:
            return

        self.__add_new_brain_line(msg)
        self.markov.begin()
        add_to_markov_dictionary(self.markov, self.chain_length, msg)
        self.markov.commit()

    def generate_sentence(self, seed_msg):
        return generate_sentence(self.markov, seed_msg, self.chain_length, self.max_words)

    def close(self):
        self.__dump_new_brain_lines()
        self.markov.close()
        os.remove(self.temp_db_file)
Example #9
0
 def load_brain(self):
     print 'Brain loading...'
     as_process(markov_dictionary_from_file, (self.temp_db_file, self.brain_file, self.chain_length)) # Shields main process from intermediate memory used
     self.markov = DatabaseDictionary(self.temp_db_file, MARKOV_VALUE_PROPS)
     print 'Brain loaded.'
     print 'Markov dictionary has %i keys' % (len(self.markov),)