docs.append(txt)
dictionary = Dictionary(docs)
id2word = dictionary.id2token
corpus = [dictionary.doc2bow(doc) for doc in docs]
lda_model = LdaMulticore(corpus=corpus,
id2word=dictionary,
random_state=100,
num_topics=5,
passes=10,
chunksize=1000,
batch=False,
alpha='asymmetric',
decay=0.5,
offset=64,
eta=None,
eval_every=0,
iterations=100,
gamma_threshold=0.001,
per_word_topics=True)
lda_model.save('models/lda_model.model')
for i in range(5):
wds = lda_model.get_topic_terms(i, 10)
wds = [e[0] for e in wds]
wds = [dictionary.id2token[e] for e in wds]
print(i, wds)
class LDAMWBase:
def __init__(self,
mtype='multiple',
resource=None,
lda_work_folder=None,
lda_model_filename=None,
lda_dict_filename=None,
lda_topic_word_count=0,
lda_topics_count=0,
resource_language=None,
data_type=None):
#
# todo Deutsch Lemmatizer / Stemmer !!!
#
self.p_stemmer = PorterStemmer()
self.wn_lemmatizer = WordNetLemmatizer()
if resource is not None:
# resource_lang == 'en' as default
resource_lang = 'en'
# hope that resource is correct and exists
if data_type == 'db':
resource_lang = Resources.select(Resources.lang).where(
Resources.resource == resource).get()
resource_lang = resource_lang.__data__['lang'].lower()
elif data_type == 'csv':
if resource_language is None:
raise Exception(
"Resource language must be defined for csv data type.")
else:
resource_lang = resource_language
else:
pass
self.stop_words = get_stop_words(resource_lang)
self.resource_identifier_name = resource
def _create_model_deps(model_name,
twordscount,
tcount,
mini=False,
mini_path=None):
if not mini:
mp = DEFAULT_PROJECT_PATH + 'topics_extractor/lda_data' + '/' + model_name
else:
mp = DEFAULT_PROJECT_PATH + 'topics_extractor/lda_data' + '/' + mini_path
mn = 'lda_model' + '_' + model_name
md = 'dictionary' + '_' + model_name
ltwordscount = twordscount
ltcount = tcount
_short_model_report = "{}{}: {} \n{}{}: {}\n{}{}: {}\n{}{}: {}\n{}{}: {}\n{}".format(
INFO_FLAG, colored("Model path", 'red', None,
['bold']), mp, INFO_FLAG,
colored("Model name", 'red', None, ['bold']), mn, INFO_FLAG,
colored("Model dictionary", 'red', None,
['bold']), md, INFO_FLAG,
colored("Topic words count", 'red', None,
['bold']), ltwordscount, INFO_FLAG,
colored("Topics count", 'red', None, ['bold']), ltcount,
"-" * 88)
if model_name != 'mini':
print(_short_model_report)
return mp, mn, md, ltwordscount, ltcount
if mtype == 'multiple':
if resource is not None:
mpath, mname, mdict, lda_topic_word_count, lda_topics_count = _create_model_deps(
self.resource_identifier_name, LDA_TOPIC_WORD_COUNT,
LDA_TOPICS_COUNT)
else:
raise Exception(
"{}Resource must be defined. Exiting... \n".format(
EXCEPTION_FLAG))
elif mtype == 'single_ltc':
mpath, mname, mdict, lda_topic_word_count, lda_topics_count = _create_model_deps(
"mini",
MINI_LDA_TOPIC_WORD_COUNT,
MINI_LDA_TOPICS_COUNT,
mini=True,
mini_path=self.resource_identifier_name + "/mini")
if lda_work_folder is None:
self.lda_work_folder = mpath
else:
self.lda_work_folder = lda_work_folder
if not os.path.exists(self.lda_work_folder):
os.mkdir(self.lda_work_folder)
if lda_model_filename is None:
self.lda_model_filename = os.path.join(self.lda_work_folder, mname)
else:
self.lda_model_filename = os.path.join(self.lda_work_folder,
lda_model_filename)
if lda_dict_filename is None:
self.lda_dict_filename = os.path.join(self.lda_work_folder, mdict)
else:
self.lda_dict_filename = os.path.join(self.lda_work_folder,
lda_dict_filename)
self.lda_topics_count = lda_topics_count
self.lda_topic_word_count = lda_topic_word_count
self.dictionary = None
self.lda_model = None
self.lda_topics = []
@staticmethod
def load_csv_data(csv_file):
df = pd.read_csv(csv_file)
train_documents = df['content'].values
return train_documents
@staticmethod
def load_single_ltc(ltc_data):
train_documents = re.split(
r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?)\s', ltc_data)
return train_documents
@staticmethod
def load_db_data(resource=None):
# if resource is None:
# art_content_stream = Articles.select()
# else:
art_content_stream = Articles.select().where(
Articles.resource == resource)
train_documents = (acs.content for acs in art_content_stream
if acs.content is not None)
return train_documents
def save_model(self,
as_name=None,
save_on_disk=True,
save_topics_into_db=False):
if save_on_disk:
print(" \t-> Model was saved as [ {} ]".format(as_name))
if as_name is not None:
self.lda_model.save(as_name)
else:
self.save_model(self.lda_model_filename)
if save_topics_into_db:
truncate_topics_tables(resource=self.resource_identifier_name)
print(" \t-> Topics will be saved in database for [ {} ]".format(
self.resource_identifier_name))
model_numbers_topics = self._get_topics()
try:
for topic_info in model_numbers_topics:
tnum = topic_info[0]
tresourceid = topic_info[1]
tname = topic_info[2]
_topic = {
'ident_number': tnum,
'value': tname,
'created_at': dt.datetime.today().date()
}
t = Topics.create(**_topic)
t_id = t.__data__['topic']
_topic_resource = {
'resource': tresourceid,
'topic': t_id,
'created_at': dt.datetime.today().date()
}
tr = TopicsResources.create(**_topic_resource)
print("{}[ {} ]".format(SUCCESS_FLAG,
self.resource_identifier_name))
except Exception as e:
print("{}{}".format(EXCEPTION_FLAG, e))
print("{}Failure: [ {} ]".format(
ERROR_FLAG, self.resource_identifier_name))
def train_model(self,
data_type,
resource,
single_ltc_data=None,
data_file_path=None,
train_corpus=None,
train_dictionary=None,
save_model_as=None,
chunksize=LDA_CHUNKSIZE,
passes=LDA_PASSES):
if train_corpus is not None:
corpus = train_corpus
elif data_type == 'db':
corpus = self._make_corpus(data_type=data_type, resource=resource)
elif data_type == 'single_ltc' and single_ltc_data is not None:
corpus = self._make_corpus(data_type=data_type,
ltc=single_ltc_data,
resource=resource)
elif data_type == 'csv' and data_file_path is not None:
corpus = self._make_corpus(data_type=data_type,
data_file_path=data_file_path,
resource=resource)
else:
raise Exception("{}Corpus is None".format(EXCEPTION_FLAG))
if train_dictionary is not None:
dictionary = train_dictionary
else:
dictionary = self.dictionary
"""
id2word parameter need to get words in topics instead of their indexes in dict
"""
_tcount = self.lda_topics_count
# self.lda_model = LdaModel(corpus=corpus, num_topics=_tcount, id2word=dictionary, passes=passes, chunksize=chunksize)
self.lda_model = LdaMulticore(corpus=corpus,
num_topics=_tcount,
id2word=dictionary,
passes=passes,
chunksize=chunksize)
if save_model_as is not None and not single_ltc_data:
self.save_model(save_model_as,
save_on_disk=True,
save_topics_into_db=False)
elif single_ltc_data:
self.save_model(self.lda_model_filename,
save_on_disk=True,
save_topics_into_db=False)
elif data_type == 'csv':
self.save_model(self.lda_model_filename,
save_on_disk=True,
save_topics_into_db=False)
else:
self.save_model(self.lda_model_filename,
save_on_disk=True,
save_topics_into_db=True)
print("{}Trained".format(SUCCESS_FLAG))
def load_model(self, model_file_path=None, dict_file_path=None):
"""
load model and dictionary from file (need to save them in train function)
uses to update model on another corpus
"""
if model_file_path is not None and os.path.exists(model_file_path):
self.lda_model = LdaMulticore.load(model_file_path)
# self.lda_model = LdaModel.load(model_file_path)
self.dictionary = Dictionary.load(dict_file_path)
print(" \t-> Loaded: [ {} ]".format(model_file_path))
elif model_file_path is None and os.path.exists(
self.lda_model_filename):
self.lda_model = LdaMulticore.load(self.lda_model_filename)
# self.lda_model = LdaModel.load(self.lda_model_filename)
self.dictionary = Dictionary.load(self.lda_dict_filename)
print(" \t-> Loaded: [ {} ]".format(self.lda_model_filename))
else:
print(
"{}Filepath you gave is incorrect. \n Give another one and retry."
"\n Exiting...".format(ERROR_FLAG))
exit()
for i in range(self.lda_model.num_topics):
terms_id = self.lda_model.get_topic_terms(
i, self.lda_topic_word_count)
terms = [self.dictionary.get(x[0]) for x in terms_id]
self.lda_topics.append(' '.join(terms))
def update_model(self,
ondata_file_path=None,
resource=None,
data_type='db'):
if ondata_file_path is not None and data_type == 'csv':
corpus = self._make_corpus(data_file_path=ondata_file_path,
data_type=data_type,
resource=resource)
elif data_type == 'db':
corpus = self._make_corpus(data_file_path=None,
data_type=data_type,
resource=resource)
else:
raise Exception("{}Corpus is None".format(EXCEPTION_FLAG))
self.lda_model.update(corpus)
def process_record(self, text, data_type):
"""
data_type - db / csv / single_ltc
"""
if data_type == 'single_ltc':
try:
self.load_model()
except Exception as e:
print("{}{}".format(EXCEPTION_FLAG, e))
pass
elif self.lda_model is None:
try:
self.load_model()
except Exception as e:
print("{}{}".format(EXCEPTION_FLAG, e))
pass
if data_type == 'db':
if self.lda_model is None:
return dict()
doc = self._prepare_single_document(text)
if doc is not None:
topics = self._get_document_topics(doc)
top_topic = topics[0]
return [('topic', self.lda_topics[top_topic])]
return [('topic', "")]
elif data_type == 'csv':
doc = self._prepare_single_document(text)
topics_in_count_by_ids = self._get_document_topics(doc)
current_doc_topic_id, current_doc_other_topics = topics_in_count_by_ids[
0], topics_in_count_by_ids[1:]
result_topic_word_descr = re.sub(
'[^A-Za-z]+', ' ', self._get_topic_by_id(current_doc_topic_id))
return [('topic', result_topic_word_descr),
('other_topics', current_doc_other_topics)]
elif data_type == 'single_ltc':
doc = self._prepare_single_document(text)
topics_in_count_by_ids = self._get_document_topics(doc)
if topics_in_count_by_ids is not None:
current_doc_topic_id, current_doc_other_topics = topics_in_count_by_ids[
0], topics_in_count_by_ids[1:]
result_topic_word_descr = re.sub(
'[^A-Za-z]+', ' ',
self._get_topic_by_id(current_doc_topic_id))
return result_topic_word_descr, current_doc_other_topics
else:
return "", []
def _get_metric_fields(self):
if self.lda_model is None:
return []
else:
return ['topic']
def _get_document_topics(self, doc, count=5):
if doc is not None:
bow = self.dictionary.doc2bow(doc)
topics = self.lda_model.get_document_topics(
bow, minimum_probability=0.0)
topics_in_count = list(
ident_number for (ident_number, prob) in sorted(
topics, key=itemgetter(1), reverse=True)[:count])
return topics_in_count
def _get_document_topic(self, doc_topics):
topic_id_probs = {}
for t_prob in doc_topics:
topic_id_probs[t_prob[0]] = t_prob[1]
doc_topic_id = sorted(topic_id_probs,
key=topic_id_probs.get,
reverse=True)[0]
doc_topic_prob = topic_id_probs[doc_topic_id]
return [doc_topic_id, doc_topic_prob]
def _prepare_single_document(self, sd):
if sd is None or type(sd) == np.float:
return None
try:
sd = sd.lower()
sd = nltk.tokenize.word_tokenize(sd)
sd = (word for word in sd if word.isalpha() and len(word) > 2)
stopped_sd = (word for word in sd if word not in self.stop_words)
lemmatized_doc = [
self.wn_lemmatizer.lemmatize(word) for word in stopped_sd
]
return lemmatized_doc
except AttributeError as e:
print("{}{}".format(EXCEPTION_FLAG, e))
return None
def _make_bow(self, text):
if text is not None:
d = self._prepare_single_document(text)
return self.dictionary.doc2bow(d)
def _make_corpus(self,
data_type,
resource,
data_file_path=None,
save_train_dict=True,
save_dict_as=None,
ltc=None):
"""
data type can be csv or db # or new - single_ltc
"""
if data_type == 'db':
documents = self.load_db_data(resource=resource)
elif data_type == 'csv' and data_file_path is not None:
documents = self.load_csv_data(data_file_path)
elif data_type == 'single_ltc' and ltc is not None:
ltc_text = " ".join(e if type(e) is str else "" for e in ltc)
documents = self.load_single_ltc(ltc_text)
else:
documents = None
print("{}documents is None. Exiting ... \n".format(ERROR_FLAG))
exit()
with Pool() as pool:
processed_docs = pool.imap(self._prepare_single_document,
documents)
pool.close()
pool.join()
processed_docs = (i for i in processed_docs if i is not None)
self.dictionary = Dictionary(processed_docs)
if save_train_dict and save_dict_as is None:
self.dictionary.save(self.lda_dict_filename)
else:
self.dictionary.save(save_dict_as)
corpus = [
self.dictionary.doc2bow(proc_doc) for proc_doc in processed_docs
]
return corpus
def _get_topic_by_id(self, topic_id):
if self.lda_topic_word_count is not None:
return self.lda_model.print_topic(topic_id,
self.lda_topic_word_count)
else:
return self.lda_model.print_topic(topic_id, 6)
def _get_topics(self, default_view=False, for_db=True):
"""
2-tuples (probability * word) of most probable words in topics
num_topics=-1 <--- to print all topics
"""
def _get_words(probabilities_words_string):
_pre_topic_with_digits_trash = " ".join(
re.findall(ALL_CHARS, probabilities_words_string))
probaply_clean_topic = re.sub(r'\b\d+(?:\.\d+)?\s+', "",
_pre_topic_with_digits_trash)
return probaply_clean_topic # " ".join(re.findall('[a-zA-Z]+', probabilities_words_string))
if default_view:
return self.lda_model.print_topics(num_topics=-1)
if for_db:
resource_id = Resources.select().where(
Resources.resource == self.resource_identifier_name).first()
resource_id = resource_id.__data__['resource']
return [(elem[0], resource_id, _get_words(elem[1]))
for elem in self.lda_model.print_topics(
num_topics=self.lda_topics_count,
num_words=self.lda_topic_word_count)]
return [(elem[0], _get_words(elem[1]))
for elem in self.lda_model.print_topics(
num_topics=self.lda_topics_count,
num_words=self.lda_topic_word_count)]
