texts_out = []

for sent in texts:

doc = nlp(" ".join(sent))

texts_out.append(" ".join(

[token.lemma_ if token.lemma_ not in ['-PRON-'] else '' for token in doc if token.pos_ in allowed_postags]))

keywords = np.array(vectorizer.get_feature_names())

topic_keywords = []

for topic_weights in lda_model.components_:

top_keyword_locs = (-topic_weights).argsort()[:n_words]

topic_keywords.append(keywords.take(top_keyword_locs))

data_words = list(sent_to_words(data))

# Do lemmatization keeping only Noun, Adj, Verb, Adverb

data_lemmatized = lemmatization(data_words, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

# print(data_lemmatized)

vectorizer = CountVectorizer(analyzer='word',

min_df=10, # minimum reqd occurences of a word

stop_words='english', # remove stop words

lowercase=True, # convert all words to lowercase

token_pattern='[a-zA-Z0-9]{3,}', # num chars > 3

# max_features=50000, # max number of uniq words

)

data_vectorized = vectorizer.fit_transform(data_lemmatized)

# Materialize the sparse data

data_dense = data_vectorized.todense()

# Compute Sparsicity = Percentage of Non-Zero cells

print("Sparsicity: ", ((data_dense > 0).sum() / data_dense.size) * 100, "%")

vectorizer, data_vectorized = get_vectorized_data(data)

# Define Search Param

search_params = {'n_components': [10, 15, 20, 25, 30, 35, 40], 'learning_decay': [0.1, 0.3, .5, .7, .9]}

# Init the Model

lda = LatentDirichletAllocation()

# Init Grid Search Class

model = GridSearchCV(lda, param_grid=search_params, cv=3, iid=True)

# Do the Grid Search

model.fit(data_vectorized)

# Best Model

best_lda_model = model.best_estimator_

print(best_lda_model)

# Model Parameters

print("Best Model's Params: ", model.best_params_)

# Log Likelihood Score

print("Best Log Likelihood Score: ", model.best_score_)

# Perplexity

print("Model Perplexity: ", best_lda_model.perplexity(data_vectorized))

# Create Document - Topic Matrix

lda_output = best_lda_model.transform(data_vectorized)

# column names

topicnames = ["Topic" + str(i) for i in range(best_lda_model.n_components)]

# index names

docnames = ["Doc" + str(i) for i in range(len(data))]

# Make the pandas dataframe

df_document_topic = pd.DataFrame(np.round(lda_output, 3), columns=topicnames, index=docnames)

# Get dominant topic for each document

dominant_topic = np.argmax(df_document_topic.values, axis=1)

df_document_topic['dominant_topic'] = dominant_topic

# Apply Style

df_document_topics = df_document_topic.head(15).style.applymap(color_green).applymap(make_bold)

print(df_document_topics)

df_topic_distribution = df_document_topic['dominant_topic'].value_counts().reset_index(name="Num Documents")

df_topic_distribution.columns = ['Topic Num', 'Num Documents']

print(df_topic_distribution)

# pyLDAvis.enable_notebook()

# panel = pyLDAvis.sklearn.prepare(best_lda_model, data_vectorized, vectorizer, mds='tsne', sort=False)

# Topic-Keyword Matrix

df_topic_keywords = pd.DataFrame(best_lda_model.components_)

# Assign Column and Index

df_topic_keywords.columns = vectorizer.get_feature_names()

df_topic_keywords.index = topicnames

# View

df_topic_keywords.head()

topic_keywords = show_topics(vectorizer=vectorizer, lda_model=best_lda_model, n_words=10)

# Topic - Keywords Dataframe

df_topic_keywords = pd.DataFrame(topic_keywords)

df_topic_keywords.columns = ['Word ' + str(i) for i in range(df_topic_keywords.shape[1])]

df_topic_keywords.index = ['Topic ' + str(i) for i in range(df_topic_keywords.shape[0])]

_, data_vectorized = get_vectorized_data(data)

# Build LDA Model

lda_model = LatentDirichletAllocation(n_components=20, # Number of topics

max_iter=10, # Max learning iterations

learning_method='online',

random_state=100, # Random state

batch_size=128, # n docs in each learning iter

evaluate_every=-1, # compute perplexity every n iters, default: Don't

n_jobs=-1, # Use all available CPUs

)

lda_output = lda_model.fit_transform(data_vectorized)

print(lda_output)

# Log Likelyhood: Higher the better

print("Log Likelihood: ", lda_model.score(data_vectorized))

# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)

print("Perplexity: ", lda_model.perplexity(data_vectorized))

# See model parameters