def save(self, *args, **kwargs): try: if not self.pk: try: word_value = kwargs.pop('word_value', None) if word_value: #print 'FileWord save - checking word' self.word = Word.get_word(word_value) except: pass except Exception as e: print_error(e) super(FileWord, self).save(*args, **kwargs)
def loadtext(self, fulltext): try: print_log('Start load text for file: ' + str(self.id)) lines = fulltext.splitlines() c_lines = 1 c_words = 1 c_sentence = 1 filewords = [] add_string = '' words = Word.objects.all().values_list('value', 'id') dwords = dict(list(words)) for line in lines: wordset = line.split() add_string += '\n' #filewords = [] for word_value in wordset: new_word = FileWord(file=self) new_word.lineno = c_lines new_word.wordno = c_words new_word.sentenceno = c_sentence new_word.word_original = add_string + word_value.decode( "utf-8") add_string = '' word_fixed = Word.fix_word_str(word_value) # Check the dictionary of words before adding word to DB if dwords.get(word_fixed): #print 'exists - ' + str(word_fixed) word_id = dwords[word_fixed] else: #print 'CREATING - ' + str(word_fixed) word_id = Word.get_word(word_fixed).id dwords[word_fixed] = word_id new_word.word_id = word_id #Word.get_word(word_fixed) #Word.get_word(re.sub('[^A-Za-z0-9]+', '', word_value)) #new_word.save() filewords.append(new_word) c_words += 1 if Word.is_end_of_sentence(word_value): c_sentence += 1 if len(filewords) > 998: try: FileWord.objects.bulk_create(filewords) except MySQLdb.Warning, e: print_error(e, level='DEBUG') pass filewords = [] c_lines += 1 print_log('End load text for file: ' + str(self.id)) self.fileloaded = True self.save()