for sentence in sentences:
X.append(sent_vectorizer(sentence, model))
neigh = NearestNeighbors(n_neighbors=5)
nbrs = neigh.fit(X)
distances, indices = nbrs.kneighbors(X)
distances = np.sort(distances, axis=0)
distances = distances[:,1]
plt.plot(distances)
plt.show()
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
model = TSNE(n_components=2, random_state=0)
np.set_printoptions(suppress=True)
Y=model.fit_transform(X)
dbscan = DBSCAN(metric='cosine', eps=0.7, min_samples=5) # you can change these parameters, given just for example
cluster_labels = dbscan.fit_predict(X)
core_samples_mask = np.zeros_like(dbscan.labels_, dtype=bool)
core_samples_mask[dbscan.core_sample_indices_] = True
labels=dbscan.labels_
for index, sentence in enumerate(sentences):
if (labels[index]==1):
print(labels[index],sentence)
plt.scatter(Y[:, 0], Y[:, 1], c=cluster_labels, cmap="plasma")
plt.show()
