def recover_term_topic_matrix(Q, anchors, tol=TOL, betaloss=1): """ Compute C such that C * Q_anchors = Q_bar minimized with Kullback-Leibler divergence. All rowsums of this matrix product are 1, for Q_* by construction, for C it follows. Params: Q: numpy float array, word coocurrence matrix anchors: list of indices of anchor words tol: tolerance for nmf beta_loss: 1 for Kullback-Leibler (more precise), 2 for L2 loss (faster) Returns: A: term x topic matrix C: intermediate result n_iter: number of iterations till convergence in computation of C """ n_topics = len(anchors) P_w = Q.sum(axis=1) Q_bar = normalize(Q, axis=1, norm='l1') Q_anchors = Q_bar[anchors, :] # Q_anchor # C Q_bar C, _, n_iter = non_negative_factorization(Q_bar, W=None, H=Q_anchors, n_components=n_topics, update_H=False, solver='mu', beta_loss=beta_loss, tol=tol) A_prime = np.multiply(P_w.reshape(-1, 1), C) A = normalize(A_prime, axis=0, norm='l1') return A, C, n_iter
def _nmf(self, X, nmf_kwargs, topic_labels=None): """ Parameters ---------- X : pandas.DataFrame, Normalized counts dataFrame to be factorized. nmf_kwargs : dict, Arguments to be passed to ``non_negative_factorization`` """ (W, H, niter) = non_negative_factorization(X.values, **nmf_kwargs) usages = pd.DataFrame(W, index=X.index, columns=topic_labels) spectra = pd.DataFrame(H, columns=X.columns, index=topic_labels) #Sort by overall usage, and rename topics with 1-indexing. topic_order = spectra.sum(axis=1).sort_values(ascending=False).index spectra = spectra.loc[topic_order, :] usages = usages.loc[:, topic_order] if topic_labels is None: spectra.index = np.arange(1, nmf_kwargs['n_components'] + 1) usages.columns = np.arange(1, nmf_kwargs['n_components'] + 1) return spectra, usages
def test_non_negative_factorization_consistency(): # Test that the function is called in the same way, either directly # or through the NMF class A = np.abs(random_state.randn(10, 10)) A[:, 2 * np.arange(5)] = 0 for solver in ('pg', 'cd'): W_nmf, H, _ = nmf.non_negative_factorization( A, solver=solver, random_state=1, tol=1e-2) W_nmf_2, _, _ = nmf.non_negative_factorization( A, H=H, update_H=False, solver=solver, random_state=1, tol=1e-2) model_class = nmf.NMF(solver=solver, random_state=1, tol=1e-2) W_cls = model_class.fit_transform(A) W_cls_2 = model_class.transform(A) assert_array_almost_equal(W_nmf, W_cls, decimal=10) assert_array_almost_equal(W_nmf_2, W_cls_2, decimal=10)
def extractActivation(y, W, w=d_w, h=d_h): """ important : in sklearn, H is "dictionary", while W is "activation". but in our case, W is "dictionary". So we have to pass W as H into sklearn """ S = librosa.core.stft(y, n_fft=w, hop_length=h) activation, components, n_iter = nmf.non_negative_factorization(X=np.abs(S.T), H=W.T, update_H=False, n_components=W.shape[1], beta=beta, max_iter=max_iter) return activation.T
def _nmf(self, X, nmf_kwargs): """ Parameters ---------- X : pandas.DataFrame, Normalized counts dataFrame to be factorized. nmf_kwargs : dict, Arguments to be passed to ``non_negative_factorization`` """ (usages, spectra, niter) = non_negative_factorization(X, **nmf_kwargs) return (spectra, usages)
def test_non_negative_factorization_consistency(): # Test that the function is called in the same way, either directly # or through the NMF class A = np.abs(random_state.randn(10, 10)) A[:, 2 * np.arange(5)] = 0 for solver in ('pg', 'cd'): W_nmf, H, _ = nmf.non_negative_factorization(A, solver=solver, random_state=1, tol=1e-2) W_nmf_2, _, _ = nmf.non_negative_factorization(A, H=H, update_H=False, solver=solver, random_state=1, tol=1e-2) model_class = nmf.NMF(solver=solver, random_state=1, tol=1e-2) W_cls = model_class.fit_transform(A) W_cls_2 = model_class.transform(A) assert_array_almost_equal(W_nmf, W_cls, decimal=10) assert_array_almost_equal(W_nmf_2, W_cls_2, decimal=10)
def update_reviews_with_topics(self, records): corpora = \ [' '.join(record[Constants.BOW_FIELD]) for record in records] document_term_matrix = \ self.tfidf_vectorizer.transform(corpora) document_topic_matrix, _, _ = nmf.non_negative_factorization( document_term_matrix, H=self.topic_term_matrix, init='nndsvd', n_components=self.num_topics, regularization='both', max_iter=Constants.TOPIC_MODEL_ITERATIONS, update_H=False) for record_index in range(len(records)): record = records[record_index] record[Constants.TOPICS_FIELD] = \ [(i, document_topic_matrix[record_index][i]) for i in range(self.num_topics)]
def _nmf(self, X, nmf_kwargs): """ Parameters ---------- X : pandas.DataFrame, Normalized counts dataFrame to be factorized. nmf_kwargs : dict, Arguments to be passed to ``non_negative_factorization`` """ # remove parameter not used by non_negative_factorization if 'cell_sampling_fraction' in nmf_kwargs: del nmf_kwargs['cell_sampling_fraction'] (usages, spectra, niter) = non_negative_factorization(X, **nmf_kwargs) return (spectra, usages)
def transform(self, doc_term_mat, tol=TOL, beta_loss=1): """ Params: doc_term_mat: scipy.sparse matrix as from CountVectorizer tol: tolerance for nmf beta_loss: 1 for Kullback-Leibler (more precise), 2 for L2 loss (faster) Returns: W_T: topic x term matrix """ M = doc_term_mat.T # product matrix M.T is document term matrix # A.T # W.T M.T W_T, _, self.n_iter_transform = non_negative_factorization( M.T, W=None, H=self.A.T, n_components=self.n_topics, update_H=False, solver='mu', beta_loss=beta_loss, tol=tol) return W_T
def update_reviews_with_topics(self, records): corpora = \ [' '.join(record[Constants.BOW_FIELD]) for record in records] document_term_matrix = \ self.tfidf_vectorizer.transform(corpora) document_topic_matrix, _, _ = nmf.non_negative_factorization( document_term_matrix, H=self.topic_term_matrix, init='nndsvd', n_components=self.num_topics, regularization='both', max_iter=Constants.TOPIC_MODEL_ITERATIONS, update_H=False) for record_index in range(len(records)): record = records[record_index] record[Constants.TOPICS_FIELD] = \ [(i, document_topic_matrix[record_index][i]) for i in range(self.num_topics)]
def estTransOrder(points, users, cluster_centers): t_hr_file = 'data\\trans_hr.txt' c_order_file = 'data\\clus_order.txt' c_time_file = 'data\\clus_time.txt' if os.path.isfile(t_hr_file) and os.path.isfile( c_order_file) and os.path.isfile(c_time_file): global trans_hr global clus_order global clus_time trans_hr = np.loadtxt(t_hr_file) clus_order = np.loadtxt(c_order_file) clus_time = np.loadtxt(c_time_file) print 'load trans/clus time, order score' else: transLen = 0.0 transTimeLen = 0.0 global trans_hr global clus_order transTimes = [ ] #number of times between each pair of clusters(no direction) for i in range(clus_k): trans_hr.append(np.zeros(clus_k)) clus_order.append(np.zeros(clus_k) * 50) trans_hr = transTimes = np.array(trans_hr) clus_order = np.array(clus_order) global clus_time for i in range(clus_k): clus_time.append(np.zeros(24) * 50) clus_time = np.array(clus_time) allusers = np.unique(points[:, 1]) for user in users: user_points = points[:, 1] == user #for all points of the user, sort by time(index 2-7) tsorted = np.array( sorted(points[user_points], key=itemgetter(2, 3, 4, 5, 6, 7))) if len(tsorted) > 0: #dates of all posts of user datesToClus = np.vstack( {tuple(row) for row in tsorted[:, 2:5]}) #remove duplicate date for date in datesToClus: #locations of the user visited in this date(boolean list) same_date = tsorted[:, 2:5] == date sd = [] for d in same_date: sd.append(d.all()) same_date = np.array(sd, dtype=bool) #locations of posts in this date l = len(tsorted[same_date]) for i in range(l - 1): #find locations that have diff clus to next locaiton thisClus = (tsorted[same_date])[i][-2] nextClus = (tsorted[same_date])[i + 1][-2] if (thisClus < clus_k): clus_time[thisClus][( tsorted[same_date])[i][5]] += 1 if thisClus != nextClus and (nextClus < clus_k): hr1 = (tsorted[same_date])[i][5] + ( tsorted[same_date] )[i][6] / 60.0 #time of prior post hr2 = (tsorted[same_date])[i + 1][5] + ( tsorted[same_date] )[i + 1][6] / 60.0 #time of latter post thisLoc = ((tsorted[same_date])[i][10], (tsorted[same_date])[i][9]) nextLoc = ((tsorted[same_date])[i + 1][10], (tsorted[same_date])[i + 1][9]) transLen += great_circle(thisLoc, nextLoc).meters transTimeLen += (hr2 - hr1) trans_hr[thisClus][nextClus] += (hr2 - hr1) trans_hr[nextClus][thisClus] += (hr2 - hr1) transTimes[thisClus][nextClus] += 1 transTimes[nextClus][thisClus] += 1 clus_order[thisClus][nextClus] += 1 #speed unit: meters/hr avgSpeed = transLen / transTimeLen trans_hr = trans_hr / transTimes for i, row in enumerate(trans_hr): for j, timeLen in enumerate(row): if j > i: if np.isnan(timeLen) and i != j: thisLoc = (cluster_centers[i][1], cluster_centers[i][0]) nextLoc = (cluster_centers[j][1], cluster_centers[j][0]) estTime = great_circle(thisLoc, nextLoc).meters / avgSpeed timeLen = estTime if round((timeLen * 100 % 100) / 50.0) == 0: trans_hr[i][j] = trans_hr[j][i] = round(timeLen) elif round((timeLen * 100 % 100) / 50.0) == 1: trans_hr[i][j] = trans_hr[j][i] = round(timeLen) + 0.5 elif round((timeLen * 100 % 100) / 50.0) == 2: trans_hr[i][j] = trans_hr[j][i] = round(timeLen) + 1 print 'transition time:' print trans_hr np.savetxt(t_hr_file, np.array(trans_hr)) clus_order = clus_order / np.max(clus_order) W, H, iter = nmf.non_negative_factorization(clus_order, n_components=10, random_state=2) clus_order = np.dot(W, H) print 'condition probability:' print clus_order np.savetxt(c_order_file, np.array(clus_order)) clus_time = clus_time / np.amax(clus_time, axis=1)[:, None] W, H, iter = nmf.non_negative_factorization(clus_time, n_components=10, random_state=2) clus_time = np.dot(W, H) print 'cluster time:' print clus_time np.savetxt(c_time_file, np.array(clus_time))
def estTransOrder(points, users, cluster_centers): t_hr_file = 'data\\trans_hr.txt' c_order_file = 'data\\clus_order.txt' c_time_file = 'data\\clus_time.txt' if os.path.isfile(t_hr_file) and os.path.isfile(c_order_file) and os.path.isfile(c_time_file): global trans_hr global clus_order global clus_time trans_hr = np.loadtxt(t_hr_file) clus_order = np.loadtxt(c_order_file) clus_time = np.loadtxt(c_time_file) print 'load trans/clus time, order score' else: transLen = 0.0 transTimeLen = 0.0 global trans_hr global clus_order transTimes = [] #number of times between each pair of clusters(no direction) for i in range(clus_k): trans_hr.append(np.zeros(clus_k)) clus_order.append(np.zeros(clus_k)*50) trans_hr = transTimes = np.array(trans_hr) clus_order = np.array(clus_order) global clus_time for i in range(clus_k): clus_time.append( np.zeros(24)*50) clus_time = np.array(clus_time) allusers = np.unique(points[:,1]) for user in users: user_points = points[:,1] == user #for all points of the user, sort by time(index 2-7) tsorted = np.array(sorted(points[user_points], key=itemgetter(2,3,4,5,6,7))) if len(tsorted)>0: #dates of all posts of user datesToClus = np.vstack({tuple(row) for row in tsorted[:,2:5]})#remove duplicate date for date in datesToClus: #locations of the user visited in this date(boolean list) same_date = tsorted[:,2:5] == date sd = [] for d in same_date: sd.append(d.all()) same_date = np.array(sd, dtype=bool) #locations of posts in this date l = len(tsorted[same_date]) for i in range(l-1): #find locations that have diff clus to next locaiton thisClus = (tsorted[same_date])[i][-2] nextClus = (tsorted[same_date])[i+1][-2] if (thisClus < clus_k): clus_time[thisClus][(tsorted[same_date])[i][5]] += 1 if thisClus != nextClus and (nextClus < clus_k): hr1 = (tsorted[same_date])[i][5] + (tsorted[same_date])[i][6]/60.0 #time of prior post hr2 = (tsorted[same_date])[i+1][5] + (tsorted[same_date])[i+1][6]/60.0 #time of latter post thisLoc = ((tsorted[same_date])[i][10], (tsorted[same_date])[i][9]) nextLoc = ((tsorted[same_date])[i+1][10], (tsorted[same_date])[i+1][9]) transLen += great_circle(thisLoc, nextLoc).meters transTimeLen += (hr2-hr1) trans_hr[thisClus][nextClus] += (hr2-hr1) trans_hr[nextClus][thisClus] += (hr2-hr1) transTimes[thisClus][nextClus] += 1 transTimes[nextClus][thisClus] += 1 clus_order[thisClus][nextClus] += 1 #speed unit: meters/hr avgSpeed = transLen / transTimeLen trans_hr = trans_hr / transTimes for i, row in enumerate(trans_hr): for j, timeLen in enumerate(row): if j > i: if np.isnan(timeLen) and i != j: thisLoc = (cluster_centers[i][1], cluster_centers[i][0]) nextLoc = (cluster_centers[j][1], cluster_centers[j][0]) estTime = great_circle(thisLoc, nextLoc).meters / avgSpeed timeLen = estTime if round( (timeLen*100%100) / 50.0) == 0: trans_hr[i][j] = trans_hr[j][i] = round(timeLen) elif round( (timeLen*100%100) / 50.0) == 1: trans_hr[i][j] = trans_hr[j][i] = round(timeLen) + 0.5 elif round( (timeLen*100%100) / 50.0) == 2: trans_hr[i][j] = trans_hr[j][i] = round(timeLen) + 1 print 'transition time:' print trans_hr np.savetxt(t_hr_file, np.array(trans_hr)) clus_order = clus_order / np.max(clus_order) W, H, iter=nmf.non_negative_factorization(clus_order, n_components=10,random_state=2) clus_order = np.dot(W,H) print 'condition probability:' print clus_order np.savetxt(c_order_file, np.array(clus_order)) clus_time = clus_time / np.amax(clus_time, axis = 1)[:, None] W, H, iter=nmf.non_negative_factorization(clus_time, n_components=10,random_state=2) clus_time = np.dot(W,H) print 'cluster time:' print clus_time np.savetxt(c_time_file, np.array(clus_time))