def save(self, *args, **kwargs):
try:
if not self.pk:
try:
word_value = kwargs.pop('word_value', None)
if word_value:
#print 'FileWord save - checking word'
self.word = Word.get_word(word_value)
except:
pass
except Exception as e:
print_error(e)
super(FileWord, self).save(*args, **kwargs)
def loadtext(self, fulltext):
try:
print_log('Start load text for file: ' + str(self.id))
lines = fulltext.splitlines()
c_lines = 1
c_words = 1
c_sentence = 1
filewords = []
add_string = ''
words = Word.objects.all().values_list('value', 'id')
dwords = dict(list(words))
for line in lines:
wordset = line.split()
add_string += '\n'
#filewords = []
for word_value in wordset:
new_word = FileWord(file=self)
new_word.lineno = c_lines
new_word.wordno = c_words
new_word.sentenceno = c_sentence
new_word.word_original = add_string + word_value.decode(
"utf-8")
add_string = ''
word_fixed = Word.fix_word_str(word_value)
# Check the dictionary of words before adding word to DB
if dwords.get(word_fixed):
#print 'exists - ' + str(word_fixed)
word_id = dwords[word_fixed]
else:
#print 'CREATING - ' + str(word_fixed)
word_id = Word.get_word(word_fixed).id
dwords[word_fixed] = word_id
new_word.word_id = word_id #Word.get_word(word_fixed) #Word.get_word(re.sub('[^A-Za-z0-9]+', '', word_value))
#new_word.save()
filewords.append(new_word)
c_words += 1
if Word.is_end_of_sentence(word_value): c_sentence += 1
if len(filewords) > 998:
try:
FileWord.objects.bulk_create(filewords)
except MySQLdb.Warning, e:
print_error(e, level='DEBUG')
pass
filewords = []
c_lines += 1
print_log('End load text for file: ' + str(self.id))
self.fileloaded = True
self.save()
